#!/usr/bin/python

import re
import time
import sqlite3
import ftplib
import tempfile
from functools import partial
from itertools import islice
from PyPDF2 import PdfFileReader

def flip(f): return lambda x, y: f(y, x)
def pack(x): return x if type(x) is tuple else (x,)
def ifNone(x, y): return x if x is not None else y

def compose(*functions):
  fs = filter(lambda x: x is not None, functions)
  return reduce(
    lambda acc, f: lambda *y: f(*pack(acc(*pack(y)))), reversed(fs), lambda *x: x)

def rasterize(blob, page):
  try:
    print "Rasterizing p. %02d" % page
    from subprocess import Popen, PIPE
    args = args = """gs -q
      -dSAFER -dBATCH -dNOPAUSE
      -sDEVICE=jpeg -dJPEGQ=95 -r150
      -sOutputFile=%%stdout
      -dFirstPage=%d -dLastPage=%d -""" % (page, page)
    p = Popen(args.split(), stdin=PIPE, stdout=PIPE)
    out, err = p.communicate(blob)
    return out
  except:
    print "Error encountered while rasterizing p. %02d" % page
    return None

def sqlite(database, timeout=10):
  try:
    sqldb = sqlite3.connect(database, timeout=timeout)
    sqldb.execute("PRAGMA foreign_keys=1")

    sqls = []
    sqls.append("""CREATE TABLE IF NOT EXISTS logs (
      id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
      created_on DATETIME,
      deleted_on DATETIME,
      name TEXT,
      pages INTEGER,
      size INTEGER,
      source_id INTEGER)""")
    sqls.append("""CREATE TABLE IF NOT EXISTS sources (
      id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
      created_on DATETIME,
      name TEXT UNIQUE,
      pages INTEGER,
      raw_source BLOB)""")
    sqls.append("""CREATE TABLE IF NOT EXISTS previews (
      id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
      document_id INTEGER,
      page INTEGER,
      format TEXT,
      raw_source BLOB,
      FOREIGN KEY(document_id) REFERENCES sources(id) ON DELETE CASCADE)""")
    sqls.append("""CREATE VIEW IF NOT EXISTS documents AS
      SELECT id, created_on, name, pages, LENGTH(raw_source) size
      FROM sources
      ORDER BY id""")
    sqls.append("""CREATE VIEW IF NOT EXISTS last_document AS
      SELECT * FROM documents
      WHERE id IN (SELECT MAX(id) FROM sources WHERE raw_source IS NOT NULL)""")
    sqls.append("""CREATE VIEW IF NOT EXISTS to_process AS
      SELECT name FROM documents
      WHERE id NOT IN (SELECT DISTINCT document_id FROM previews)""")
    sqls.append("""CREATE VIEW IF NOT EXISTS to_delete AS
      SELECT name FROM documents
      WHERE (julianday(datetime('now')) - julianday(substr(created_on, 0, 20))) > 1.0""")

    map(sqldb.execute, sqls)
    sqldb.commit()
  except:
    print "Error encountered while setting up local database"
    sqldb.rollback()
    raise
  else:
    return sqldb

def timestamp():
  return time.strftime("%Y-%m-%d %H:%M:%S%z")

def vacuum(sqldb):
  try:
    sqls = []
    sqls.append("""UPDATE logs
      SET deleted_on='%s', source_id=NULL
      WHERE deleted_on IS NULL
      AND name IN (SELECT name FROM to_delete)""" % timestamp())
    sqls.append("DELETE FROM sources WHERE name IN (SELECT name FROM to_delete)")
    sqls.append("VACUUM")
    map(sqldb.execute, sqls)
    sqldb.commit()
  except:
    print "Error encountered while vacuuming database"
    sqldb.rollback()
  finally:
    pass

def documents(sqldb, count):
  sql = """SELECT id, created_on, name, pages, raw_source FROM sources
    WHERE raw_source IS NOT NULL ORDER BY id DESC LIMIT %d"""
  c = sqldb.execute(sql % count)
  return c.fetchall()

def document_by_id(sqldb, document_id=None):
  sql = """SELECT id, name, pages, raw_source FROM sources
    WHERE id=ifnull(?, (SELECT MAX(id) FROM documents)) LIMIT 1"""
  c = sqldb.execute(sql, [document_id])
  return c.fetchone()

def document_by_name(sqldb, file_name):
  sql = """SELECT id, name, pages, raw_source FROM sources WHERE name=? LIMIT 1"""
  c = sqldb.execute(sql, [file_name])
  return c.fetchone()

def preview_by_id(sqldb, document_id, page_id):
  sql = """SELECT id, format, raw_source FROM previews
    WHERE document_id=? AND page=? LIMIT 1"""
  c = sqldb.execute(sql, [document_id, page_id])
  return c.fetchone()

def to_process(sqldb):
  sql = """SELECT name FROM to_process"""
  c = sqldb.execute(sql)
  return c.fetchall()

def download(ftp, file_name, *args):
  try:
    print "Downloading %s ..." % file_name
    tf = tempfile.TemporaryFile()
    ftp.retrbinary("RETR %s" % file_name, tf.write)
    tf.seek(0)
    blob = tf.read()
    tf.seek(0)
    pdf = PdfFileReader(tf)
    pages = pdf.numPages
    tf.close()
  except:
    print "Error downloading %s" % file_name
    return (file_name, None, None)
  else:
    return (file_name, pages, blob)

def log(sqldb, file_name, pages=None, blob=None):
  try:
    print "Logging %s ..." % file_name
    sql = """INSERT INTO logs (created_on, name, pages, size, source_id) VALUES
      ('%s', ?, ?, ?, (SELECT id FROM sources WHERE name=?))""" % timestamp()
    sqldb.execute(sql, [file_name, pages, len(blob) if blob is not None else None, file_name])
    sqldb.commit()

    return (file_name, pages, blob)
  except:
    print "Error logging %s" % file_name
  finally:
    pass

def write(sqldb, file_name, pages=None, blob=None):
  try:
    print "Writing %s ..." % file_name
    sql = """INSERT OR REPLACE INTO sources (created_on, name, pages, raw_source)
      VALUES ('%s', ?, ?, ?)""" % timestamp()
    sqldb.execute(sql, [file_name, pages, sqlite3.Binary(blob) if blob is not None else None])
    sqldb.commit()

    return (file_name, pages, blob)
  except:
    print "Error writing %s" % file_name
  finally:
    pass

def exists(sqldb, file_name):
  try:
    print "Checking whether %s exists ..." % file_name
    sql = """SELECT COUNT(DISTINCT id) FROM sources WHERE name=?"""
    c = sqldb.execute(sql, [file_name])
    (n,) = c.fetchone()
    return n > 0
  except:
    print "Error checking whether %s exists" % file_name
    return False
  finally:
    pass


def process(sqldb, file_name, pages, blob):
  try:
    print "Processing file %s ... " % file_name
    previews = map(lambda p: (p, rasterize(blob, p)), range(1, pages + 1))

    sql = """INSERT INTO previews (document_id, page, format, raw_source)
      VALUES ((SELECT id FROM sources WHERE name=?), ?, ?, ?)"""
    w = partial(sqldb.execute, sql)
    map(lambda (p, b): w([file_name, p, "jpeg", sqlite3.Binary(b)]), previews)

    sqldb.commit()
    return (file_name, pages, blob)
  except:
    print "Error encountered while processing %s" % file_name
    sqldb.rollback()
  finally:
    pass

def rename(ftp, file_name, destination):
  try:
    print "Renaming %s to %s" % (file_name, destination)
    ftp.rename(file_name, "%s/%s" % (destination, file_name))
  except:
    print "Error encountered while renaming file %s" % file_name
  finally:
    pass

def get_document_by_name(database, file_name):
  sqldb = sqlite(database)
  with sqldb:
    return document_by_name(sqldb, file_name)

def get_document_by_id(database, document_id):
  sqldb = sqlite(database)
  with sqldb:
    return document_by_id(sqldb, document_id)

def get_last_document(database):
  sqldb = sqlite(database)
  with sqldb:
    return document_by_id(sqldb)

def get_last_documents(database, count=10):
  sqldb = sqlite(database)
  with sqldb:
    return documents(sqldb, count)

def get_preview(database, document_id, page_id):
  sqldb = sqlite(database)
  with sqldb:
    return preview_by_id(sqldb, document_id, page_id)

def fetch(database, host, username, password, location="/", archive=None, pattern="", limit=0):
  try:
    print "Fetching %d first element(s) matching %s" % (limit, pattern)
    print "Connecting to %s as %s" % (host, username)
    ftp = ftplib.FTP(host, username, password)
    map(ftp.cwd, filter(lambda x: len(x) > 0, location.split('/')))
    print "Listing files ..."
    files = ftp.nlst()

    sqldb = sqlite(database)

    matches = (f for f in files
      if re.compile(pattern, re.IGNORECASE).search(f) and not exists(sqldb, f))
    matches = islice(matches, limit)

    d = partial(download, ftp)
    l = partial(log, sqldb)
    w = partial(write, sqldb)
    p = partial(process, sqldb)
    docs = map(compose(l, p, w, d, w), matches)

    r = compose(partial(flip(partial(rename, ftp)), archive), lambda x: x[0])
    if archive is not None:
      map(r, docs)

  except ftplib.error_perm, resp:
    if str(resp) == "550 No files found":
      print("No such directory")
    return 1
#  except:
#    print "Unexpected error encountered"
#    return 2
  else:
    return docs
  finally:
    print "Cleaning up ..."
    vacuum(sqldb)
    ftp.close()
    sqldb.close()

if __name__ == "__main__":
  import argparse

  parser = argparse.ArgumentParser(description='Fetch new scanned documents', add_help=False)
  parser.add_argument('--help', action='help', help='Show this help message and exit')
  parser.add_argument('-h', '--host', metavar='h', type=str,
    help='Hostname')
  parser.add_argument('-u', '--username', metavar='u', type=str, default="anonymous",
    help='Username (default: anonymous)')
  parser.add_argument('-p', '--password', metavar='p', type=str, default="",
    help='Password (default: '')')
  parser.add_argument('-l', '--location', metavar='l', type=str, default="/",
    help="Location (default: '/')")
  parser.add_argument('-a', '--archive', metavar='a', type=str,
    help="Archive location (default: location)")
  parser.add_argument('-m', '--match', metavar='m', type=str, default=".pdf$",
    help='Pattern of files to fetch (default: .pdf$)')
  parser.add_argument('-d', '--database', metavar='d', type=str, default="documents.sqlite3",
    help='SQLite3 database file (default: documents.sqlite)')
  parser.add_argument('--limit', metavar='n', type=int, default=10,
    help='Limit of new documents to fetch (default: 10, max: 100)')

  args = parser.parse_args()
  r = fetch(args.database,
    args.host, args.username, args.password,
    args.location, args.archive, args.match, min(args.limit, 100))
  exit(0 if type(r) is list else r)