# Copyright (c) 2009 ActiveState Software Inc.
# See http://www.activestate.com/activepython/license/ for licensing
# information.

"""
    pypm.common.repository
    ~~~~~~~~~~~~~~~~~~~~~~

    Code related to repository functions
"""

import os
from os.path import join, exists, normpath, abspath, relpath, dirname
import logging
import gzip
import hashlib
from contextlib import closing, nested
import tempfile
import atexit
import shutil
import urllib2
from urlparse import urlparse
from ConfigParser import SafeConfigParser
from fnmatch import fnmatch

from pypm.common import supported, db, console, net, python, log, activestate
import pypm.common.cmdln_branches_cfg as cmdln
from pypm.common.util import (xjoin, url_join, require_option, dlocked,
                              BareDateTime, find_in_directory)
from pypm.common.net import URLGetter
from pypm.common.textui import ProgressBar
from pypm.common.package import BinaryPackage, RepoPackage

LOG = logging.getLogger(__name__)


class MultiRepositorySet(object):
    """Multiple instances of RepositorySet's
    
    List of repositories are read from the config file (etc/activestate.conf)
    which file also defines the mappings from packages to the respective
    repositories.
    """
    
    def __init__(self, path, configfile):
        self.path = path
        self._readcfg(configfile)
        
    def _readcfg(self, configfile):
        cfg = SafeConfigParser()
        cfg.read(configfile)
        
        self.reposets = {}
        for part in cfg.get('repositories', 'parts').split():
            name = cfg.get(part, 'name')
            url = cfg.get(part, 'url')
            self.reposets[name] = RepositorySet(join(self.path, name),
                                                name, url)
            
        self.mapping = []
        for line in cfg.get('packages', 'mapping').split('\n'):
            line = line.strip()
            if line.startswith('#'): continue # comment
            pattern, repo = line.split()
            self.mapping.append((pattern.strip(), repo.strip()))
            
    def get_repository(self, bpkg):
        """Return the underlying repository
        
        Pick the appropriate repository respected by the mapping
        (``self.mapping``). Name of the bpkg is used in matching the patterns
        in self.mapping. pyver and osarch of bpkg is finally used in picking up
        the underlying repository in the choosen reposet.
        
        - bpkg: An instance of ``pypm.common.package.BinaryPackage``
        """
        bpkg_fn = bpkg.make_filename()
        for pattern, name in self.mapping:
            if fnmatch(bpkg_fn, pattern):
                return self.reposets[name].get_repository(
                    bpkg.pyver, bpkg.osarch
                )
    
    def __iter__(self):
        """Iter over available repositories"""
        for name, reposet in self.reposets.items():
            for repo in reposet:
                yield repo
    
#
# RepositorySet
#

class RepositorySet(object):
    """A set of repositories

    Typically a set of repositories each corresponding to a particular
    architecture and a Python version, eg: win32-x86-py26
    """

    def __init__(self, path, name, url):
        self.path = path
        self.name = name
        self.url = url

    def create_repositories(self):
        """Create repositories for supported configuration"""
        for osarch in supported.os_architectures:
            for pyver in supported.py_versions:
                console.mkdirs(
                    join(self.path, pyver, osarch))

    def get_repository(self, pyver, osarch, autocreate=False):
        path = xjoin(self.path, pyver, osarch)
        url = '/'.join([self.url, pyver, osarch])
        if autocreate:
            # create, if does not already exist
            console.mkdirs(path)
            
        return Repository(path, self.name, pyver, osarch, url)

    def __iter__(self):
        """Iter over all supported repositories
        
        If a supported repository does not exists, simply create the repository
        directory before returning it.
        """
        for osarch in supported.os_architectures:
            for pyver in supported.py_versions:
                yield self.get_repository(pyver, osarch, autocreate=True)

    def __str__(self):
        return '{0}<{1.path}, {1.url}>'.format(self.__class__.__name__, self)
    __repr__ = __str__


#
# Repository
#

class Repository(object):
    """Repository directory containing packages"""

    def __init__(self, path, name, pyver, osarch, url):
        self.path = path
        self.name = name
        self.pyver = pyver
        self.osarch = osarch
        self.url = url

    def find_packages(self):
        """Return available packages in the repository"""
        return find_in_directory(self.path, "*.pypm")

    def find_all_packages(self):
        """Return all packages whether succeeded or failed

        Return a list of tuples of the form:

          (pkgfile, pkgdir, succeeded)

        where:

          pkgfile   - path to the package file (may not exist)
          pkgdir    - path to the package dir (.d/ directory)
          succeeded - True of the pkgfile exists
        """
        processed = set()
        for fil in find_in_directory(self.path, '*.pypm*'):
            if fil.endswith('.pypm'):
                pkgfile, pkgdir, succeeded = fil, fil + '.d', True
            elif fil.endswith('.pypm.d'):
                pkgfile, pkgdir = fil[:-2], fil
                succeeded = exists(pkgfile)
            else:
                continue
            if pkgfile not in processed:
                processed.add(pkgfile)
                yield pkgfile, pkgdir, succeeded

    def __str__(self):
        return '{0}<{1.path}>'.format(self.__class__.__name__, self)


# 
# RepositoryIndex
# 

class RepoPackageDatabase(db.SqlAlchemyDatabase):
    """A database containing instances of ``pypm.common.package.RepoPackage``"""
            
db.setup(RepoPackageDatabase, RepoPackage,
         primary_keys=['name', 'version',
                       'pyver', 'osarch',
                       'pkg_version'])

class RepositoryIndex(object):
    """Index of packages in a repository

    Repositories can optionally have index files .. which are especially useful
    when the repository is only available remotely over the wire.

    The index file contains a list of all packages along with their metadata and
    the relative location of the package file.

    There is just one index file:

      - index | index.gz - list of packages that are available (sqlite)
    """

    def __init__(self, repository):
        assert isinstance(repository, Repository)
        self.repository = repository

    def get_index(self):
        """Return an existing index as ``RepoPackageDatabase``

        Returned index database corresponds to a temporary file (as the index
        file is originally compressed; it needs to be extracted to a temporary
        location) .. hence any attempts to "write" on the returned index
        database will be futile.
        """
        return RepoPackageDatabase(_ungzip(xjoin(self.repository.path, 'index.gz')))

    def generate_indices(self):
        """Generated all index files

        Return the number of packages available in the repository
        """
        return self._generate_index_gz()

    def _generate_index_gz(self):
        """Generate `index.gz` - the repository index

        index.gz is the compressed form of `index` which is nothing but a Sqlite
        database of succeeded packages available in the repository. See
        ``RepoPackageDatabase``
        """
        assert exists(self.repository.path)
        idx_path = xjoin(self.repository.path, 'index')
        idx_gz_path = idx_path + '.gz'
        
        console.rm(idx_path)
        db = RepoPackageDatabase(idx_path, touch=True)
        
        # Tag BE packages; so client may use it to determine if a package is
        # available only to BE customers or not.
        pkgtags = 'be' if self.repository.name == 'be' else ''

        with closing(db):
            LOG.info('finding packages')
            packages = self.repository.find_packages()

            LOG.info('processing %d packages', len(packages))
            rpkg_list = [
                RepoPackage.create_from(
                    BinaryPackage.from_json(self._read_info_json(pkgfile)),
                    relpath=relpath(pkgfile, self.repository.path),
                    tags=pkgtags)
                for pkgfile in ProgressBar.iterate(packages)
            ]
            
            # Optimize index size by removing the "description" field.
            # PyPI's descriptions are typically very long - see
            # http://pypi.python.org/pypi/zc.buildout for example - hence we
            # must remove them from the index.
            for rpkg in rpkg_list:
                rpkg.description = 'N/A'

            # keep only the latest version/pkg_version in index
            LOG.info('pruning older versions')
            rpkg_list = _prune_older_binary_releases(rpkg_list)

            LOG.info('writing index')
            with db.transaction() as session:
                session.add_all(rpkg_list)
                session.commit()
                session.close()

        LOG.info('compresing index (%s)', idx_gz_path)
        console.rm(idx_gz_path)
        with closing(gzip.open(idx_gz_path, 'wb')) as f:
            f.write(open(idx_path, 'rb').read())
        console.rm(idx_path)

        return len(rpkg_list)

    @staticmethod
    def _read_info_json(pypm_file):
        """Read cached info.json from the .d/ directory"""
        info_json_loc = xjoin(pypm_file + '.d', 'info.json')

        assert exists(info_json_loc), (
            'Missing %s. It is the responsibility of pypm.builder to create '
            'a `info.json` in the .d/ directory. We cannot afford to read it '
            'from the tarball everytime (expensive).') % info_json_loc

        return open(info_json_loc).read()

def _prune_older_binary_releases(packages):
    """Prune all older releases (pkg_version) of the package"""
    releases = {}

    for pkg in packages:
        if pkg.full_name in releases:
            prevrel = releases[pkg.full_name]
            if prevrel.pkg_version < pkg.pkg_version:
                releases[pkg.full_name] = pkg
        else:
            releases[pkg.full_name] = pkg

    return releases.values()
        

#
# classes for managing remote repositories
#

class RemoteRepositorySet(object):
    """Represent a remotely available RepositorySet"""

    def __init__(self, url):
        self.url = url

    def get_repository(self, pyver, osarch):
        return RemoteRepository(url_join(self.url, [pyver, osarch]))

class RemoteRepository(object):
    """Represent a remotely available Repository"""

    # Filename of the actual remote index file
    REMOTE_INDEX_FILENAME = "index.gz"

    def __init__(self, url):
        self.url = url

    def download_index(self, target_file, force, verbose=True):
        """Download repository index

        - verbose: If False, try not to print (LOG.info) anything to console
          unless an actual download happens.
        """
        index_url = url_join(self.url, [self.REMOTE_INDEX_FILENAME])
        try:
            idxgz_file = net.download_file(index_url, dirname(target_file), {
                    'use_cache': not force,
                    'save_properties': True,
                    'info': 'Get: [%s] :repository-index:' % (
                        urlparse(self.url).netloc)
                    })
        except urllib2.HTTPError, e:
            if e.code == 304: # Not Modified
                # repository was not modified; no need to download index
                if verbose:
                    LOG.info('Hit: [%s] :repository-index:',
                             urlparse(self.url).netloc)
                return False
            elif e.code == 404: # Not Found
                raise ValueError, (
                    '{0.url} does not appear to be a valid repository '
                    'because {1} is missing.'.format(self, index_url))
            raise
        
        # extract index.gz (REMOTE_INDEX_FILENAME) to index (target_file)
        with closing(gzip.open(idxgz_file, 'rb')) as f:
            with open(target_file, 'wb') as f2:
                f2.write(f.read())
        return True

    def get_unique_id(self):
        """Return an alpha-numeric ID unique to this repository (URL)"""
        return hashlib.md5(self.url).hexdigest()

    def __str__(self):
        return '{0.__class__.__name__}<{0.url}>'.format(self)

class RemoteRepositoryManager(object):
    """Manage multiple remote repositories with a local cache"""

    def __init__(self, path):
        # local cache directory where repository indexes will be stored
        self.path = path
        console.mkdirs(path)

    def get_index_db(self, remote_repository):
        """Return the index database for remote repository

        If necessary, download the index automatically
        """
        return RepoPackageDatabase(
            self.get_local_index_path(
                remote_repository))

    def sync_repository(self, remote_repository, force, verbose=True):
        """Sync the cache for a remote repository"""
        with dlocked(self.path):
            assert isinstance(remote_repository, RemoteRepository)
            idx_path = self.get_local_index_path(remote_repository)
            console.mkdirs(dirname(idx_path))
            return remote_repository.download_index(idx_path, force, verbose)

    def get_remote_index_last_download_attempt_time(self, remote_repository):
        """Return the UTC datetime when the index file was last *attempted* to
        download

        The download may not have happened, however, due to unmodified ETag.

        If no index is available (as in, 'pypm sync' or an equivalent was never
        run in the first place), return None
        """
        original_index_file = join(
            dirname(self.get_local_index_path(remote_repository)),
            remote_repository.REMOTE_INDEX_FILENAME)
        urlprops = URLGetter.URLProperties(original_index_file).load()
        if urlprops:
            return BareDateTime.to_datetime(
                urlprops.custom['last_attempt_utc'])

    def get_local_index_path(self, remote_repository):
        return xjoin(
            self.path,
            remote_repository.get_unique_id(),
            'index')


#
# cmdln
# 

@cmdln.option('-R', '--multi-repository-set-path',
              help='MultiRepositorySet to store built packages in')
@cmdln.useconfig(
    filenames = [],
    vars = dict(NAS_DATA_PATH=activestate.NAS_DATA_PATH))
class Commands(log.LogawareCmdln):
    name = "pypm-repository"

    def initialize(self):
        require_option(self.options, 'configfile')
        require_option(self.options, 'multi_repository_set_path')
        activestate.nas_check(self.options.configfile)
        
    @cmdln.option('', '--platform', help='Only for this platform')
    def do_genidx(self, subcmd, opts):
        """${cmd_name}: Generate the index file for all available repositories

        ${cmd_usage}
        ${cmd_option_list}
        """
        with self.bootstrapped():
            logsdir = xjoin(self.options.multi_repository_set_path, '_logs')
            with log.archivedby(logging.getLogger('pypm'),
                                logsdir,
                                'repository_genidx',
                                level=logging.INFO,
                                formatter=logging.Formatter('%(asctime)s %(message)s')):
                mreposet = MultiRepositorySet(
                    self.options.multi_repository_set_path,
                    self.options.configfile
                )
                with console.long_process(
                    LOG,
                    desc='genidx for: %s' % mreposet.path,
                    begin_info={}, final_info={}):

                    for repo in mreposet:
                        if opts.platform and opts.platform not in repo.path:
                            LOG.info('skipping %s', repo)
                            continue
                        LOG.info('')
                        LOG.info('generating index for repository: '
                                 '{0.name}:{0.pyver}:{0.osarch}'.format(repo))
                        LOG.info('%s', repo.path)
                        idx = RepositoryIndex(repo)
                        idx.generate_indices()

def main():
    Commands(install_console=True).main()

        

#
# -- internal --
#

def _ungzip(path):
    """Ungzip a compressed to a temporary location

    Return the extracted location
    """
    with closing(gzip.open(path, 'rb')) as f:
        with tempfile.NamedTemporaryFile(delete=False) as f2:
            shutil.copyfileobj(f, f2)
            return f2.name
