Insight-TwitterData/TrainSaveModel.py at master · jn2clark/Insight-TwitterData · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
# -*- coding: utf-8 -*-
"""
Created on Sat Sep 12 22:03:50 2015

@author: jesseclark

Train and save the word2vec model
Takes in the processed tweets from ProcessTweets.py
Most of the params are adjustable
Will also make a hash table for quick lookup of
tweets associated with #'s.
Removing stop-words can take quite a while (1 hr for 10 M tweets)
Training is about 10-20 mins per iteration


"""

import gensim
from gensim.summarization import summarize
from gensim.summarization import keywords
import sys
from stemming.porter2 import stem
from nltk.corpus import stopwords
import itertools
from collections import Counter
from random import shuffle
import os
import sys
import imp
import HollaFunctions
import pymysql as mdb
import LoadCleanTweets

if __name__ == '__main__':

    # train from the sql db
    use_sql=True
    # db name
    dbname='Tweets'
    # table name
    table_name='table1'
    # which column
    col_name='text'

    # max number of tweets to save in the hash table
    max_term=2
    # also do a randomized training version
    do_rand=True

    # do the filterd list (stop) removed?
    do_stop =False

    # training params
    nnet_size=200#500 #200, 500?
    min_count=15 #play with this 510 or 15?
    window=10

    sample=1e-5
    iterations=10
    negative=0


    #where to save the model
    save_dir=''
    # where the data is
    data_dir=''
    # model save name
    base_name="USmodel"

    other_str='-d26'
    # where the combined text data is
    fnames=['USAcombined'+other_str+'.txt']

    # save name
    save_model_keys=base_name+'-n'+str(nnet_size)+'-mc'+str(min_count)+ \
                    '-w'+str(window)+'-i'+str(iterations)+'-ng'+str(negative)+other_str

    if not use_sql:
        # train US basic
        print("\n Loading text documents...")
        document_usa=HollaFunctions.load_text(text_name=data_dir+fnames[0])
    else:
        print("\n Loading text from SQL...")
        document_usa=HollaFunctions.get_field_sql(dbname, table_name, col_name)
    print("\n Done...")

    # load the filtered list if it exists
    #try
    # train USA with stop word removal
    if do_stop:
        print("removing stop words...(might take a while)")
        filtered_usa=[[word for word in sentance if word not in stopwords.words("english")] for sentance in document_usa]
        print('Done... \n')

    # do the hash tables now
    print('\n Preparing Hash table... ')
    htable=HollaFunctions.hash_tweets(document_usa, mterm='#')
    # save the table with the original text
    HollaFunctions.save_object(htable, save_dir+'USAhtable'+other_str+'.pkl')
    print('\n Preparing Shortened Hash tables... ')
    htable=HollaFunctions.shorten_hash_tweets(htable, max_term=max_term, shuffle_tweets=True)
    HollaFunctions.save_object(htable, save_dir+'USAhtable-short'+other_str+'.pkl')
    del htable


    if not do_rand:
        # now do training
        print("Training USA model... ")
        print(data_dir+fnames[0])
        modelUSA=HollaFunctions.train_and_save_model(document_usa, nnet_size=nnet_size,
                                           min_count=min_count, window=window, sample=sample, iterations=iterations, negative=negative,
                                           save_name=save_dir+save_model_keys)
        print(save_dir+save_model_keys)
        print(" ")

        if do_stop:
            print("Training filtered USA model... \n")
            modelUSA=HollaFunctions.train_and_save_model(filtered_usa, nnet_size=nnet_size,
                                               min_count=min_count, window=window, sample=sample, iterations=iterations, negative=negative,
                                               save_name=save_dir+save_model_keys+'Stop')
            print(save_dir+save_model_keys+'Stop')

    if do_rand:
        print("Training randomized USA model... \n")
        print(data_dir+fnames[0])

        # randomize tweet order so they are not in chrono order
        # shuffle the docs
        shuffle(document_usa)

        modelUSA=HollaFunctions.train_and_save_model(document_usa, nnet_size=nnet_size,
                                               min_count=min_count, window=window, sample=sample, iterations=iterations, negative=negative,
                                               save_name=save_dir+save_model_keys+'Rand')
        print(save_dir+save_model_keys)
        print(" ")


        if do_stop:
            # train USA with stop word removal
            print("Training randomized filtered USA model... \n")
            shuffle(filtered_usa)
            modelUSA=HollaFunctions.train_and_save_model(filtered_usa, nnet_size=nnet_size,
                                                   min_count=min_count, window=window, sample=sample, iterations=iterations, negative=negative,
                                                   save_name=save_dir+save_model_keys+'Stop'+'Rand')
            print(save_dir+save_model_keys+'Stop'+'Rand')

        print('\n Finished randomized training...')

    print('\n Finished everything!')