#!/usr/bin/python

import pymysql, os, sys, atexit, re

#MariaDB [words]> describe words;
#+-------+------------------+------+-----+---------+----------------+
#| Field | Type             | Null | Key | Default | Extra          |
#+-------+------------------+------+-----+---------+----------------+
#| id    | int(11) unsigned | NO   | PRI | NULL    | auto_increment |
#| word  | char(64)         | NO   | UNI | NULL    |                |
#| count | int(11)          | NO   |     | 0       |                |
#+-------+------------------+------+-----+---------+----------------+

sql = """INSERT INTO `words` (`word`, `count`)
        VALUES (%s, %s) ON DUPLICATE KEY 
        UPDATE `count` = `count`+VALUES(`count`) """

child_pid = 0

def pushbuf():
    print('words:', len(wbuf))
    #print('size:', sys.getsizeof(wbuf))
    global child_pid
    if child_pid != 0: os.wait()
    child_pid = os.fork()
    if child_pid != 0: 
        wbuf.clear()
        return
    with db.cursor() as cursor:
        try:
            db.begin()
            cursor.executemany(sql, wbuf.items())
            cursor.close()
            db.commit()
        except:
            db.rollback()
            raise(print('wtf1?!'))
    sys.exit(0)

def end():
    if child_pid != 0:
        f.close()
        os.wait()
        db.close()

atexit.register(end)

db = pymysql.connect("localhost","root","paSSword","words", charset="utf8")
with db.cursor() as cursor:
    try:
        db.autocommit(False)
        cursor.execute("TRUNCATE `words`")
        cursor.close()
        db.commit()
    except:
        raise(print('wtf0?!'))


wbuf={}
wbuflen=10000
f=open("warpeace.txt","r")

for line in f:
    for word in re.findall(r'[^\W\d]+', line):
        newword=word.lower()
        #wbuf[newword] = wbuf.get(newword,0)+1
        if newword in wbuf:
            wbuf[newword] += 1
        else:
            wbuf[newword] = 1
            if len(wbuf) >= wbuflen: pushbuf()
pushbuf()