Differences
This shows you the differences between two versions of the page.
Next revision | Previous revision | ||
11_26_2018 [2018/11/26 21:14] – created cynthia.kahn | 11_26_2018 [2018/11/26 21:30] (current) – cynthia.kahn | ||
---|---|---|---|
Line 1: | Line 1: | ||
===== Week Ending November 25 ===== | ===== Week Ending November 25 ===== | ||
+ | * Completed uploading entire tweets data on cynthiak DB | ||
+ | * Two new tables | ||
+ | * All python files and parsed files are in / | ||
+ | |||
+ | **//Some useful commands for handling tweets dataset//** | ||
+ | |||
+ | sed ' | ||
+ | sed -n ' | ||
+ | sed ' | ||
+ | \\ | ||
+ | **//SQL to create table users_info// | ||
+ | \\ | ||
+ | DROP TABLE IF EXISTS users_info; \\ | ||
+ | \\ | ||
+ | CREATE TABLE users_info( \\ | ||
+ | userID BIGINT, | ||
+ | userName TEXT, | ||
+ | screenName TEXT, | ||
+ | userLocation TEXT, | ||
+ | url TEXT, | ||
+ | description TEXT, | ||
+ | translatorType TEXT, | ||
+ | protected TEXT, | ||
+ | verified TEXT, | ||
+ | followersCount INTEGER, | ||
+ | friendsCount INTEGER, | ||
+ | listedCount INTEGER, | ||
+ | favouritesCount INTEGER, | ||
+ | statusesCount INTEGER, | ||
+ | userCreated VARCHAR(50), | ||
+ | utcOffset TEXT, | ||
+ | userTimeZone VARCHAR(50), | ||
+ | geoEnabled TEXT, | ||
+ | lang TEXT, | ||
+ | contributorsEnabled TEXT, | ||
+ | isTranslator TEXT, | ||
+ | profileBackgroundColor TEXT, | ||
+ | profileBackgroundImageURL TEXT, | ||
+ | profileBackgroundTile TEXT, | ||
+ | profileLinkColor TEXT, | ||
+ | profileSidebarBorderColor TEXT, | ||
+ | profileSidebarFillColor TEXT, | ||
+ | profileTextColor TEXT, | ||
+ | profileUseBackgroundImage TEXT, | ||
+ | profileImageURL TEXT, | ||
+ | defaultProfile TEXT, | ||
+ | defaultProfileImage TEXT, | ||
+ | userFollowing TEXT, | ||
+ | followRequestSent TEXT, | ||
+ | notifications TEXT | ||
+ | ); | ||
+ | |||
+ | \\ | ||
+ | psql cynthiak | ||
+ | \copy users_info FROM '/ | ||
+ | |||
+ | \\ | ||
+ | **//SQL to create table entities_info// | ||
+ | \\ | ||
+ | DROP TABLE IF EXISTS entities_info; | ||
+ | \\ | ||
+ | CREATE TABLE entities_info( \\ | ||
+ | | ||
+ | entitiesURL TEXT, | ||
+ | userMentionsID1 BIGINT, | ||
+ | userMentionsID2 BIGINT, | ||
+ | symbols TEXT, | ||
+ | entitiesMediaID BIGINT, | ||
+ | entitiesMediaIndices TEXT, | ||
+ | entitiesMediaURL TEXT, | ||
+ | entitiesType TEXT, | ||
+ | entitiesSizeSmall TEXT, | ||
+ | entitiesSizeMedium TEXT, | ||
+ | entitiesSizeThumb TEXT, | ||
+ | entitiesSizeLarge TEXT, | ||
+ | extendedEntitiesMediaID BIGINT, | ||
+ | extendedEntitiesMediaIndices TEXT, | ||
+ | extendedEntitiesMediaURL TEXT, | ||
+ | extendedDisplayURL TEXT, | ||
+ | extendedExpandedURL TEXT, | ||
+ | extendedEntitiesType TEXT, | ||
+ | extendedSizeSmall TEXT, | ||
+ | extendedSizeMedium TEXT, | ||
+ | extendedSizeThumb TEXT, | ||
+ | extendedSizeLarge TEXT | ||
+ | ); | ||
+ | |||
+ | \\ | ||
+ | psql cynthiak | ||
+ | \copy entities_info FROM '/ | ||