I wrote a quick script to count all rows in all tables in HBase. This works great for my Dev clusters that have ever-growing tables filled with clutter. The script uses a MapReduce job to go against all HBase tables. I have used this in Prod, but with mixed results: Sometimes the HBase tables are too large for the MR jobs to run within 24 hours.<\/p>\n
\r\n\r\n#!\/bin\/bash\r\n# Filename: rc-start-rowcount.sh\r\n# Description: start a row count for each table\r\n#\r\n# Example:\r\n# \/opt\/scripts\/rc-start-rowcount.sh\r\n\r\n# 1. check if the row count is already running\r\n# 2. if the row count is NOT running, then run a row count\r\n\r\ncd \/opt\/cloudera\/parcels\/CDH\/bin\r\n\r\nScriptDir="\/opt\/scripts\/";\r\nWorkingDir="\/opt\/scripts\/rc-work";\r\nTest="";\r\nListOfHBaseTables="rc-tables.txt";\r\nListOfRunningYarnJobs="rc-yarn-jobs.txt";\r\nScriptToRun="rc-script.sh";\r\nLogDir="\/var\/log\/scripts";\r\nLogFile="rc-start-rowcount.log";\r\n\r\necho "`date`: Start" >> $LogDir\/$LogFile;\r\n\r\nStartTest=`ps ax|grep rc-parserowcount.sh|grep bash`\r\necho $StartTest\r\nif [[ ! $StartTest == "" ]]; then\r\necho "`date`: WARNING: rc-parse-rowcount.sh is running, exit" >> $LogDir\/$LogFile;\r\necho $StartTest >> $LogDir\/$LogFile;\r\nexit;\r\nfi\r\n\r\n# create the script\r\necho "#!\/bin\/bash" > $WorkingDir\/$ScriptToRun\r\n\r\necho 'list; quit;' | hbase shell > $WorkingDir\/$ListOfHBaseTables\r\nsed -i '\/^$\/d' $WorkingDir\/$ListOfHBaseTables\r\nsed -i '$d' $WorkingDir\/$ListOfHBaseTables\r\n\r\n# get running applications from yarn\r\nyarn application -list > $WorkingDir\/$ListOfRunningYarnJobs\r\n\r\nwhile read table; do\r\n#echo 'table:' $table\r\n# 1. check if row count is running\r\n# if Test is blank=NOT running, anything else=running\r\nTest=`grep $table $WorkingDir\/$ListOfRunningYarnJobs`;\r\nif [[ $Test == "" ]]; then\r\n# 2. if the row count is NOT running, then run a row count\r\necho "sleep 10;hbase org.apache.hadoop.hbase.mapreduce.RowCounter $table > $WorkingDir\/$table.txt 2>&1 &" >> $WorkingDir\/$ScriptToRun 2>&1\r\n#echo 'run this table:' $table\r\necho "`date`: Process Table: $table" >> $LogDir\/$LogFile;\r\nfi\r\ndone <$WorkingDir\/$ListOfHBaseTables\r\n\r\n# set the script to be executable\r\nchmod +x $WorkingDir\/$ScriptToRun\r\n\r\n# run the script that included all map reduce jobs\r\ncat $WorkingDir\/$ScriptToRun\r\n$WorkingDir\/$ScriptToRun\r\n\r\necho "`date`: End" >> $LogDir\/$LogFile;\r\n\r\n<\/pre>\n","protected":false},"excerpt":{"rendered":"I wrote a quick script to count all rows in all tables in HBase. This works great for my Dev clusters that have ever-growing tables […]<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"jetpack_post_was_ever_published":false,"_jetpack_newsletter_access":"","_jetpack_dont_email_post_to_subs":false,"_jetpack_newsletter_tier_id":0,"_jetpack_memberships_contains_paywalled_content":false,"_jetpack_memberships_contains_paid_content":false,"footnotes":"","jetpack_publicize_message":"","jetpack_publicize_feature_enabled":true,"jetpack_social_post_already_shared":true,"jetpack_social_options":{"image_generator_settings":{"template":"highway","enabled":false},"version":2}},"categories":[88,75,25],"tags":[],"class_list":["post-1068","post","type-post","status-publish","format-standard","hentry","category-bash","category-hbase","category-linux"],"jetpack_publicize_connections":[],"jetpack_featured_media_url":"","jetpack_shortlink":"https:\/\/wp.me\/p1BQ8S-he","jetpack_sharing_enabled":true,"_links":{"self":[{"href":"https:\/\/www.developerscloset.com\/index.php?rest_route=\/wp\/v2\/posts\/1068","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.developerscloset.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.developerscloset.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.developerscloset.com\/index.php?rest_route=\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/www.developerscloset.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1068"}],"version-history":[{"count":1,"href":"https:\/\/www.developerscloset.com\/index.php?rest_route=\/wp\/v2\/posts\/1068\/revisions"}],"predecessor-version":[{"id":1069,"href":"https:\/\/www.developerscloset.com\/index.php?rest_route=\/wp\/v2\/posts\/1068\/revisions\/1069"}],"wp:attachment":[{"href":"https:\/\/www.developerscloset.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1068"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.developerscloset.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1068"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.developerscloset.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1068"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}