#
# client.py - Max Usachev <maxusachev@gmail.com>
#             Ed Bartosh <bartosh@gmail.com>
#             Peter Bienstman <Peter.Bienstman@UGent.be>
#

import os
import socket
import urllib
import tarfile
import httplib

from partner import Partner
from text_formats.xml_format import XMLFormat
from utils import traceback_string, SyncError, SeriousSyncError

# Avoid delays caused by Nagle's algorithm.
# http://www.cmlenz.net/archives/2008/03/python-httplib-performance-problems

realsocket = socket.socket
def socketwrap(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0):
    sockobj = realsocket(family, type, proto)
    sockobj.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
    return sockobj
socket.socket = socketwrap

# Buffer the response socket.
# http://mail.python.org/pipermail/python-bugs-list/2006-September/035156.html
# Fix included here for systems running Python 2.5.

class HTTPResponse(httplib.HTTPResponse):

    def __init__(self, sock, **kw):
        httplib.HTTPResponse.__init__(self, sock, **kw)
        self.fp = sock.makefile("rb") # Was unbuffered: sock.makefile("rb", 0)

httplib.HTTPConnection.response_class = HTTPResponse

# Register binary formats.

from binary_formats.mnemosyne_format import MnemosyneFormat
BinaryFormats = [MnemosyneFormat]


class Client(Partner):

    program_name = "unknown-SRS-app"
    program_version = "unknown"
    BUFFER_SIZE = 8192
    # The capabilities supported by the client. Note that we assume that the
    # server supports "mnemosyne_dynamic_cards".
    capabilities = "mnemosyne_dynamic_cards"  # "facts", "cards"
    # The following setting can be set to False to speed up the syncing
    # process on e.g. mobile clients where the media files don't get edited
    # externally.
    check_for_edited_local_media_files = True
    # Setting the following to False will speed up the initial sync, but in that
    # case the client will not have access to all of the review history in order
    # to e.g. display statistics. Also, it will not be possible to keep the
    # local database when there are sync conflicts. If a client makes it
    # possible for the user to change this value, doing so should result in
    # redownloading the entire database from scratch.
    interested_in_old_reps = True
    # Store prerendered question, answer and tag fields in database. The only
    # benefit of this is fast operation for a 'browse cards' dialog which
    # directly operates at the SQL level. If you don't use this, set to False
    # to reduce the database size.
    store_pregenerated_data = True
    # On mobile clients with slow SD cards copying a large database for the
    # backup before sync can take longer than the sync itself, so we offer
    # reckless users the possibility to skip this.
    do_backup = True
    # Setting this to False will leave all the uploading of anonymous science
    # logs to the sync server. Recommended to set this to False for mobile
    # clients, which are not always guaranteed to have an internet connection.
    upload_science_logs = True

    def __init__(self, machine_id, database, ui):
        Partner.__init__(self, ui)
        self.machine_id = machine_id
        self.database = database
        self.text_format = XMLFormat()
        self.server_info = {}
        self.con = None
        self.behind_proxy = None  # Explicit variable for testability.
        self.proxy = None

    def request_connection(self):

        """If we are not behind a proxy, create the connection once and reuse
        it for all requests. If we are behind a proxy, we need to revert to
        HTTP 1.0 and use a separate connection for each request.

        """

        # If we haven't done so, determine whether we're behind a proxy.
        if self.behind_proxy is None:
            import urllib
            proxies = urllib.getproxies()
            if "http" in proxies:
                self.behind_proxy = True
                self.proxy = proxies["http"]
            else:
                self.behind_proxy = False
        # Create a new connection or reuse an existing one.
        if self.behind_proxy:
            httplib.HTTPConnection._http_vsn = 10
            httplib.HTTPConnection._http_vsn_str = "HTTP/1.0"
            if self.proxy is not None:
                self.con = httplib.HTTPConnection(self.proxy, self.port)
            else:  # Testsuite has set self.behind_proxy to True to simulate
                # being behind a proxy.
                self.con = httplib.HTTPConnection(self.server, self.port)
        else:
            httplib.HTTPConnection._http_vsn = 11
            httplib.HTTPConnection._http_vsn_str = "HTTP/1.1"
            if not self.con:
                self.con = httplib.HTTPConnection(self.server, self.port)

    def url(self, url_string):
        if self.behind_proxy and self.proxy:
            url_string = self.server + ":/" + url_string
        return url_string

    def sync(self, server, port, username, password):
        try:
            self.server = socket.gethostbyname(server)
            self.port = port
            if self.do_backup:
                self.ui.set_progress_text("Creating backup...")
                backup_file = self.database.backup()
            # We check if files were edited outside of the program. This can
            # generate EDITED_MEDIA_FILES log entries, so it should be done
            # first.
            if self.check_for_edited_local_media_files:
                self.ui.set_progress_text("Checking for edited media files...")
                self.database.check_for_edited_media_files()
            # Always create media files, otherwise they are not synced across.
            self.ui.set_progress_text("Dynamically creating media files...")
            self.database.dynamically_create_media_files()
            # Set timeout long enough for e.g. a slow NAS waking from 
            # hibernation.
            socket.setdefaulttimeout(60)
            self.login(username, password)
            # Generating media files at the server side could take some time,
            # so we update the timeout.
            self.con = None
            socket.setdefaulttimeout(15*60)
            self.get_server_check_media_files()
            # Do a full sync after either the client or the server has restored
            # from a backup.
            if self.database.is_sync_reset_needed(\
                self.server_info["machine_id"]) or \
                self.server_info["sync_reset_needed"] == True:
                self.resolve_conflicts(restored_from_backup=True)
            # First sync, fetch database from server.
            elif self.database.is_empty():
                self.get_server_media_files()
                if self.server_info["supports_binary_transfer"]:
                    self.get_server_entire_database_binary()
                else:
                    self.get_server_entire_database()
                self.get_sync_finish()
                # Fetch config settings.
                self.login(username, password)
                self.get_server_generate_log_entries_for_settings()
                self.get_server_log_entries()
                self.get_sync_finish()
            # First sync, put binary database to server if supported.
            elif not self.database.is_empty() and \
                    self.server_info["is_database_empty"] and \
                    self.supports_binary_upload():
                self.put_client_media_files(reupload_all=True)
                self.put_client_entire_database_binary()
                self.get_sync_finish()
                # Upload config settings.
                self.login(username, password)
                self.database.generate_log_entries_for_settings()
                self.put_client_log_entries()
                self.get_server_log_entries()
                self.get_sync_finish()
            else:
                # Upload local changes and check for conflicts.
                result = self.put_client_log_entries()
                if result == "OK":
                    self.put_client_media_files()
                    self.get_server_media_files()
                    self.get_server_log_entries()
                    self.get_sync_finish()
                else:
                    self.resolve_conflicts()
            self.ui.show_information("Sync finished!")
        except Exception, exception:
            self.ui.close_progress()
            serious = True
            if type(exception) == type(socket.gaierror()):
                self.ui.show_error("Could not find server!")
                serious = False
            elif type(exception) == type(socket.error()):
                self.ui.show_error("Could not connect to server!")
                serious = False
            elif type(exception) == type(socket.timeout()):
                self.ui.show_error("Timeout while waiting for server!")
            elif type(exception) == type(SyncError()):
                self.ui.show_error(str(exception))
                serious = False
            elif type(exception) == type(SeriousSyncError()):
                self.ui.show_error(str(exception))
            else:
                self.ui.show_error(traceback_string())
            if serious and self.do_backup:
                # Only serious errors should result in the need for a full
                # sync next time.
                self.ui.show_error("Sync failed, restoring from backup. " + \
                    "The next sync will need to be a full sync.")
                if backup_file:
                    self.database.restore(backup_file)
        finally:
            if self.con:
                self.con.close()
            self.ui.close_progress()

    def supports_binary_upload(self):
        return self.capabilities == "mnemosyne_dynamic_cards" and \
            self.interested_in_old_reps and self.store_pregenerated_data \
            and self.program_name == self.server_info["program_name"] and \
            self.program_version == self.server_info["program_version"]

    def resolve_conflicts(self, restored_from_backup=False):
        if restored_from_backup:
            message = "The database was restored from a backup, either " + \
                "automatically because of an aborted sync or manually by " + \
                "the user.\nFor safety, a full sync needs to happen and " + \
                "you need to choose which copy of the database to keep and " + \
                "which copy to discard.\n"
        else:
            message = "Conflicts detected during sync! Choose which version to keep and which version to discard."
        # Ask for conflict resolution direction.
        if self.supports_binary_upload():
            result = self.ui.show_question(message,
                "Keep local version", "Fetch remote version", "Cancel")
            results = {0: "KEEP_LOCAL", 1: "KEEP_REMOTE", 2: "CANCEL"}
            result = results[result]
        else:
            message += " " + "Your client only stores part of the server " + \
                "database or uses a different software version, " + \
                "so you can only fetch the remote version."
            result = self.ui.show_question(message,
                "Fetch remote version", "Cancel", "")
            results = {0: "KEEP_REMOTE", 1: "CANCEL"}
            result = results[result]
        # Keep remote. Reset the partnerships afterwards, such that syncing
        # with a third party will also trigger a full sync.
        if result == "KEEP_REMOTE":
            self.get_server_media_files(redownload_all=True)
            if self.server_info["supports_binary_transfer"]:
                self.get_server_entire_database_binary()
            else:
                self.get_server_entire_database()
            self.database.reset_partnerships()
            self.get_sync_finish()
        # Keep local.
        elif result == "KEEP_LOCAL":
            self.put_client_media_files(reupload_all=True)
            self.put_client_entire_database_binary()
            self.get_sync_finish()
        # Cancel.
        elif result == "CANCEL":
            self.get_sync_cancel()

    def _check_response_for_errors(self, response, can_consume_response=True):
        # Check for non-Mnemosyne error messages.
        if response.status != httplib.OK:
            raise SeriousSyncError("Internal server error:\n" + response.read())
        if can_consume_response == False:
            return
        # Check for Mnemosyne error messages.
        message, traceback = self.text_format.parse_message(response.read())
        if "server error" in message.lower():
            raise SeriousSyncError(message + "\n" + traceback)

    def login(self, username, password):
        self.ui.set_progress_text("Logging in...")
        client_info = {}
        client_info["username"] = username
        client_info["password"] = password
        client_info["user_id"] = self.database.user_id()
        client_info["machine_id"] = self.machine_id
        client_info["program_name"] = self.program_name
        client_info["program_version"] = self.program_version
        client_info["database_name"] = self.database.name()
        client_info["database_version"] = self.database.version
        client_info["capabilities"] = self.capabilities
        client_info["partners"] = self.database.partners()
        client_info["interested_in_old_reps"] = self.interested_in_old_reps
        client_info["store_pregenerated_data"] = self.store_pregenerated_data
        client_info["upload_science_logs"] = self.upload_science_logs
        # Signal if the database is empty, so that the server does not give a
        # spurious sync cycle warning if the client database was reset.
        client_info["is_database_empty"] = self.database.is_empty()
        # Not yet implemented: preferred renderer.
        client_info["render_chain"] = ""
        # Add optional program-specific information.
        client_info = self.database.append_to_sync_partner_info(client_info)
        # Try to establish a connection, but don't force a restore from backup
        # if we can't login.
        try:
            self.request_connection()
            self.con.request("PUT", self.url("/login"),
                self.text_format.repr_partner_info(client_info).\
                encode("utf-8") + "\n")
            response = self.con.getresponse()
        except Exception, e:
            raise SyncError("Could not connect to server!")
        # Check for errors, but don't force a restore from backup if we can't
        # login.
        try:
            self._check_response_for_errors(\
                response, can_consume_response=False)
        except SeriousSyncError:
            raise SyncError("Logging in: server error.")
        response = response.read()
        if "message" in response:
            message, traceback = self.text_format.parse_message(response)
            message = message.lower()
            if "server error" in message:
                raise SyncError("Logging in: server error.")
            if "access denied" in message:
                raise SyncError("Wrong username or password.")
            if "cycle" in message:
                raise SyncError(\
                    "Sync cycle detected. Sync through intermediate partner.")
            if "same machine ids" in message:
                raise SyncError(\
"You have manually copied the data directory before sync. Sync needs to start from an empty database.")
        self.server_info = self.text_format.parse_partner_info(response)
        self.database.set_sync_partner_info(self.server_info)
        if self.database.is_empty():
            self.database.change_user_id(self.server_info["user_id"])
        elif self.server_info["user_id"] != client_info["user_id"] and \
            self.server_info["is_database_empty"] == False:
            raise SyncError("Error: mismatched user ids.\n" + \
                "The first sync should happen on an empty database.\n" + \
                "Backup then delete the local database and try again.")
        if self.server_info["database_version"] != client_info["database_version"]:
            raise SyncError("Error: database version mismatch.\n" + \
                "Upgrade client and server to latest version.")
        if self.server_info["program_version"] != client_info["program_version"]:
            raise SyncError("Error: Mnemosyne version mismatch.\n" + \
                "Upgrade client and server to latest version.")           
        self.database.create_if_needed_partnership_with(\
            self.server_info["machine_id"])
        self.database.merge_partners(self.server_info["partners"])

    def get_server_check_media_files(self):
        self.ui.set_progress_text("Asking server to check for updated media files...")
        self.request_connection()
        self.con.request("GET", self.url(\
            "/server_check_media_files?" + \
            "session_token=%s" % (self.server_info["session_token"], )))
        response = self.con.getresponse()
        self._check_response_for_errors(response, can_consume_response=True)

    def put_client_log_entries(self):

        """Contrary to binary files, the size of the log is not known until we
        create it. In order to save memory on mobile devices, we don't want to
        construct the entire log in memory before sending it on to the server.
        However, chunked uploads are in the grey area of the WSGI spec and are
        also not supported by older HTTP 1.0 proxies (e.g. Squid before 3.1).
        Therefore, as a compromise, rather then streaming chunks in a single
        message, we break up the entire log in different messages of size
        self.BUFFER_SIZE.

        """

        number_of_entries = self.database.number_of_log_entries_to_sync_for(\
            self.server_info["machine_id"])
        if number_of_entries == 0:
            return
        self.ui.set_progress_text("Sending log entries...")
        self.ui.set_progress_range(number_of_entries)
        self.ui.set_progress_update_interval(number_of_entries/20)
        buffer = ""
        count = 0
        for log_entry in self.database.log_entries_to_sync_for(\
                self.server_info["machine_id"]):
            buffer += self.text_format.repr_log_entry(log_entry)
            count += 1
            self.ui.increase_progress(1)
            if len(buffer) > self.BUFFER_SIZE or count == number_of_entries:
                buffer = \
                    self.text_format.log_entries_header(number_of_entries) \
                    + buffer + self.text_format.log_entries_footer()
                self.request_connection()
                self.con.request("PUT", self.url(\
                    "/client_log_entries?session_token=%s" \
                    % (self.server_info["session_token"],)),
                    buffer.encode("utf-8"))
                buffer = ""
                response = self.con.getresponse()
                self._check_response_for_errors(response,
                    can_consume_response=False)
                response = response.read()
                message, traceback = self.text_format.parse_message(response)
                message = message.lower()
                if "server error" in message:
                    raise SeriousSyncError(message)
                if "conflict" in message:
                    return "conflict"
        return "OK"

    def put_client_entire_database_binary(self):
        self.ui.set_progress_text("Sending entire binary database...")
        for BinaryFormat in BinaryFormats:
            binary_format = BinaryFormat(self.database)
            if binary_format.supports(self.server_info["program_name"],
                self.server_info["program_version"],
                self.server_info["database_version"]):
                assert self.store_pregenerated_data == True
                assert self.interested_in_old_reps == True
                filename = binary_format.binary_filename(\
                    self.store_pregenerated_data, self.interested_in_old_reps)
                break
        self.request_connection()
        self.con.putrequest("PUT",
                self.url("/client_entire_database_binary?session_token=%s" \
                % (self.server_info["session_token"], )))
        self.con.putheader("content-length", os.path.getsize(filename))
        self.con.endheaders()
        for buffer in self.stream_binary_file(filename):
            self.con.send(buffer)
        binary_format.clean_up()
        self._check_response_for_errors(self.con.getresponse())

    def _download_log_entries(self, stream):
        element_loop = self.text_format.parse_log_entries(stream)
        number_of_entries = int(element_loop.next())
        if number_of_entries == 0:
            return
        self.ui.set_progress_range(number_of_entries)
        self.ui.set_progress_update_interval(number_of_entries/50)
        for log_entry in element_loop:
            self.database.apply_log_entry(log_entry)
            self.ui.increase_progress(1)
        self.ui.set_progress_value(number_of_entries)

    def get_server_log_entries(self):
        self.ui.set_progress_text("Getting log entries...")
        if self.upload_science_logs:
            self.database.dump_to_science_log()
        self.request_connection()
        self.con.request("GET", self.url(\
            "/server_log_entries?session_token=%s" \
            % (self.server_info["session_token"], )))
        response = self.con.getresponse()
        self._check_response_for_errors(response, can_consume_response=False)
        self._download_log_entries(response)
        # The server will always upload the science logs of the log events
        # which originated at the server side.
        self.database.skip_science_log()

    def get_server_entire_database(self):
        self.ui.set_progress_text("Getting entire database...")
        filename = self.database.path()
        # Create a new database. Note that this also resets the
        # partnerships, as required.
        self.database.new(filename)
        self.request_connection()
        self.con.request("GET", self.url("/server_entire_database?" + \
            "session_token=%s" % (self.server_info["session_token"], )))
        response = self.con.getresponse()
        self._check_response_for_errors(response, can_consume_response=False)
        self._download_log_entries(response)
        self.database.load(filename)
        # The server will always upload the science logs of the log events
        # which originated at the server side.
        self.database.skip_science_log()
        # Since we start from a new database, we need to create the
        # partnership again.
        self.database.create_if_needed_partnership_with(\
            self.server_info["machine_id"])

    def get_server_entire_database_binary(self):
        self.ui.set_progress_text("Getting entire binary database...")
        filename = self.database.path()
        self.database.abandon()
        self.request_connection()
        self.con.request("GET", self.url(\
            "/server_entire_database_binary?" + \
            "session_token=%s" % (self.server_info["session_token"], )))
        response = self.con.getresponse()
        self._check_response_for_errors(response, can_consume_response=False)
        file_size = int(response.getheader("mnemosyne-content-length"))
        self.download_binary_file(response, filename, file_size)
        self.database.load(filename)
        self.database.create_if_needed_partnership_with(\
            self.server_info["machine_id"])
        self.database.remove_partnership_with(self.machine_id)

    def get_server_generate_log_entries_for_settings(self):
        self.ui.set_progress_text("Getting settings...")
        self.request_connection()
        self.con.request("GET", self.url(\
            "/server_generate_log_entries_for_settings?" + \
            "session_token=%s" % (self.server_info["session_token"], )))
        response = self.con.getresponse()
        self._check_response_for_errors(response, can_consume_response=True)

    def put_client_media_files(self, reupload_all=False):
        self.ui.set_progress_text("Sending media files...")
        if reupload_all:
            filenames = list(self.database.all_media_filenames())
        else:
            filenames = list(self.database.media_filenames_to_sync_for(\
                self.server_info["machine_id"]))
        total_size = 0
        for filename in filenames:
            total_size += os.path.getsize(os.path.join(\
                self.database.media_dir(), filename))
        self.ui.set_progress_range(total_size)
        self.ui.set_progress_update_interval(total_size/50)
        for filename in filenames:
            self.request_connection()
            self.con.putrequest("PUT",
                self.url("/client_media_file?session_token=%s&filename=%s" \
                % (self.server_info["session_token"],
                urllib.quote(filename.encode("utf-8"), ""))))
            full_path = os.path.join(self.database.media_dir(), filename)
            file_size = os.path.getsize(full_path)
            self.con.putheader("content-length", file_size)
            self.con.endheaders()
            for buffer in self.stream_binary_file(full_path, progress_bar=False):
                self.con.send(buffer)
                self.ui.increase_progress(len(buffer))
            self._check_response_for_errors(self.con.getresponse())
        self.ui.set_progress_value(total_size)

    def get_server_media_files(self, redownload_all=False):
        self.ui.set_progress_text("Getting list of media files to download...")
        # Get list of names of all media files to download.
        media_url = "/server_media_filenames?session_token=%s" \
            % (self.server_info["session_token"], )
        if redownload_all:
             media_url += "&redownload_all=1"
        self.request_connection()
        self.con.request("GET", self.url(media_url))
        response = self.con.getresponse()
        self._check_response_for_errors(response, can_consume_response=False)
        total_size = int(response.getheader("mnemosyne-content-length"))
        if total_size == 0:
            # Make sure to read the full message, even if it's empty,
            # since we reuse our connection.
            response.read()
            return
        # Download each media file.
        self.ui.set_progress_text("Getting media files...")
        self.ui.set_progress_range(total_size)
        self.ui.set_progress_update_interval(total_size/50)
        for filename in response.read().split("\n"):
            filename = unicode(filename, "utf-8")
            self.request_connection()
            self.con.request("GET",
                self.url("/server_media_file?session_token=%s&filename=%s" \
                % (self.server_info["session_token"],
                urllib.quote(filename.encode("utf-8"), ""))))
            response = self.con.getresponse()
            self._check_response_for_errors(response,
                can_consume_response=False)
            file_size = int(response.getheader("mnemosyne-content-length"))
            # Make sure a malicious server cannot overwrite anything outside
            # of the media directory.
            filename = filename.replace("../", "").replace("..\\", "")
            filename = filename.replace("/..", "").replace("\\..", "")
            filename = os.path.join(self.database.media_dir(), filename)
            self.download_binary_file(response, filename,
                                      file_size, progress_bar=False)
            self.ui.increase_progress(file_size)
        self.ui.set_progress_value(total_size)

    def get_sync_cancel(self):
        self.ui.set_progress_text("Cancelling sync...")
        self.request_connection()
        self.con.request("GET", self.url("/sync_cancel?session_token=%s" \
            % (self.server_info["session_token"], )),
            headers={"connection": "close"})
        self._check_response_for_errors(self.con.getresponse())

    def get_sync_finish(self):
        self.ui.set_progress_text("Finishing sync...")
        self.request_connection()
        self.con.request("GET", self.url("/sync_finish?session_token=%s" \
            % (self.server_info["session_token"], )),
            headers={"connection": "close"})
        self._check_response_for_errors(self.con.getresponse())
        # Only update after we are sure there have been no errors.
        self.database.update_last_log_index_synced_for(\
            self.server_info["machine_id"])
