joh3 thoughts, musings, and what not coding related, or not what

BigData commands

cd ~
cd bin
./quickstart.sh
cd ../data
unzip -p zipcodes.wkt.zip | hdfs dfs -put - /zipcodes/zipcodes.wkt
#all_trips prepared sepately
cat all_trips | hdfs dfs -put - /trips/all_trips
hdfs dfs -mkdir /jars
hdfs dfs -put *.jar /jars
./beeline_connect.sh

      #setup initial 
      !run function-ddl.sql
      !run make_trips_table.hql

Setup Trips and then trip_bins

# trip_table.sql
drop table if exists trips_tab;
create table if not exists trips_tab
stored as parquet as
select ST_AsText(ST_Point(plon,plat)) as WKT from trips
where ((plat > 40 and plat <41) and (plon > -75 and plon < -71)) 
limit 20000;

drop table if exists trip_bins;
create table trip_bins
stored as parquet as
select ST_AsText(ST_BinEnvelope(0.001, t.bin_id)) shape,count(1) as population from (
select ST_Bin(0.001, ST_Point(plon,plat)) bin_id from trips where ((plat > 40 and plat <41) and (plon > -75 and plon < -71))) as t
group by t.bin_id;