Gofedlib setup

.gitignore

@@ -1,2 +1,4 @@
 *.pyc
 *.swp
+build
+gofedlib.egg-info

gofedlib-cli

@@ -0,0 +1,115 @@
+#!/bin/env python
+
+import json
+from argparse import ArgumentParser
+from gofedlib.go.functions import api, project_packages
+from gofedlib.go.importpath.decomposerbuilder import ImportPathsDecomposerBuilder
+from gofedlib.go.importpath.normalizer import ImportPathNormalizer
+from gofedlib.go.importpath.parserbuilder import ImportPathParserBuilder
+from gofedlib.providers.providerbuilder import ProviderBuilder
+
+
+def dict2json(o, pretty=True):
+    if pretty is True:
+        return json.dumps(o, sort_keys=True, separators=(',', ': '), indent=2)
+    else:
+        return json.dumps(o)
+
+
+def ippaths2providers(deps_list):
+    normalizer = ImportPathNormalizer()
+    dependencies = map(lambda l: normalizer.normalize(l), deps_list)
+    dependencies = list(set(dependencies))
+
+    decomposer = ImportPathsDecomposerBuilder().buildLocalDecomposer()
+    decomposer.decompose(dependencies)
+
+    project_provider = ProviderBuilder().buildUpstreamWithLocalMapping()
+
+    result = []
+    for prefix in decomposer.classes().keys():
+        if prefix != 'Native' and prefix != 'Unknown':
+            result.append(project_provider.parse(prefix).prefix())
+
+    return result
+
+
+def get_dependencies(packages, selection):
+    dependencies = {}
+
+    if selection['packages']:
+        dependencies['deps-packages'] = []
+        for dependency in packages['dependencies']:
+            for item in dependency['dependencies']:
+                dependencies['deps-packages'].append(item['name'])
+
+    if selection['main']:
+        dependencies['deps-main'] = []
+        for record in packages['main']:
+            for dependency in record['dependencies']:
+                dependencies['deps-main'].append(dependency)
+
+    if selection['tests']:
+        dependencies['deps-tests'] = []
+        for record in packages['tests']:
+            for dependency in record['dependencies']:
+                dependencies['deps-tests'].append(dependency)
+
+    for key in dependencies.keys():
+        dependencies[key] = ippaths2providers(dependencies[key])
+        dependencies[key] = map(lambda l: 'https://%s' % l, dependencies[key])
+
+    return dependencies
+
+
+if __name__ == '__main__':
+    ap = ArgumentParser()
+    ap.add_argument('path')
+    ap.add_argument('-m', '--dependencies-main', help="list of dependencies in main packages",
+                    default=False, action='store_true')
+    ap.add_argument('-d', '--dependencies-packages', help="list of dependencies in packages",
+                    default=False, action='store_true')
+    ap.add_argument('-t', '--dependencies-tests', help="list of dependencies in tests",
+                    default=False, action='store_true')
+    ap.add_argument('-a', '--api', help="API listing",
+                    default=False, action='store_true')
+    ap.add_argument('-j', '--pretty', help="print JSON nicely formatted",
+                    default=False, action='store_true')
+    ap.add_argument('-p', '--packages', help="packages listing",
+                    default=False, action='store_true')
+    ap.add_argument('-o', '--output', help="output file",
+                    default=None, type=str)
+    ap.add_argument('-r', '--raw-project-packages', help="raw output from gofedlib",
+                    default=False, action='store_true')
+
+    args = ap.parse_args()
+    result = {}
+    project_package_info = None
+
+    if args.raw_project_packages:
+        project_package_info = project_packages(args.path)
+        result['raw-project-packages'] = project_package_info
+
+    if args.packages:
+        if not project_package_info:
+            packages_info = project_packages(args.path)
+        result['packages'] = packages_info['packages']
+
+    if args.api:
+        result['api'] = api(args.path)
+
+    if args.dependencies_main \
+            or args.dependencies_tests or args.dependencies_packages:
+        selection = {
+            'main': args.dependencies_main,
+            'packages': args.dependencies_packages,
+            'tests': args.dependencies_tests
+        }
+
+        result.update(get_dependencies(project_packages(args.path), selection))
+
+    if args.output:
+        with open(args.output, 'w') as f:
+            f.write(dict2json(result, args.pretty))
+    else:
+        print(dict2json(result, args.pretty))

__init__.py → gofedlib/__init__.py

@@ -1,4 +1,4 @@
-import sys 
+import sys
 
-if __name__ == "gofed_lib":
+if __name__ == "gofedlib":
 	sys.modules['lib'] = sys.modules[__name__]

config/config.py → gofedlib/config/config.py

@@ -1,5 +1,5 @@
 import ConfigParser
-from gofed_lib.utils import getScriptDir
+from gofedlib.utils import getScriptDir
 import os
 
 class Config(object):

logger/logging.yaml → gofedlib/logger/logging.yaml

@@ -1,4 +1,4 @@
---- 
+---
 version: 1
 formatters:
         simpleFormatter:
@@ -18,12 +18,12 @@ loggers:
         pkgdb_client:
                 level: INFO
                 handlers: [consoleHandler]
-                qualname: gofed_lib.distribution.clients.pkgdb.client
+                qualname: gofedlib.distribution.clients.pkgdb.client
                 propagate: 0
         distribution_capturer:
                 level: INFO
                 handlers: [consoleHandler]
-                qualname: gofed_lib.distribution.eco.capturer
+                qualname: gofedlib.distribution.eco.capturer
                 propagate: 0
         distribution_snapshot_capturer:
                 level: INFO

projectsignature/parser.py → gofedlib/projectsignature/parser.py

@@ -8,8 +8,8 @@
 #
 
 import re
-from gofed_lib.providers.providerbuilder import ProviderBuilder
-from gofed_lib.distribution.distributionnameparser import DistributionNameParser
+from gofedlib.providers.providerbuilder import ProviderBuilder
+from gofedlib.distribution.distributionnameparser import DistributionNameParser
 
 class ProjectSignatureParser(object):
 

urlbuilder/builder.py → gofedlib/urlbuilder/builder.py

@@ -5,7 +5,7 @@
 # - upstream repository clone url (github.com, bitbucket.org)
 #
 
-from gofed_lib.distribution.helpers import Build, Rpm
+from gofedlib.distribution.helpers import Build, Rpm
 
 class UrlBuilder(object):
 

setup.py

@@ -0,0 +1,84 @@
+#!/usr/bin/python
+
+import os
+import subprocess
+from setuptools import setup, find_packages
+from distutils.command.install import install as DistutilsInstall
+
+
+class GofedlibInstall(DistutilsInstall):
+
+    def _compile_parsego(self):
+        path = os.path.join("gofedlib", "go", "symbolsextractor")
+        cmd = "go build -o %s %s" % (os.path.join(path,
+                                                  "parseGo"), os.path.join(path, "parseGo.go"))
+
+        process = subprocess.Popen(
+            cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = process.communicate()
+        retval = process.returncode
+
+        if retval != 0:
+            raise RuntimeError(stderr)
+
+    def run(self):
+        self._compile_parsego()
+        return DistutilsInstall.run(self)
+
+
+def get_requirements():
+    with open('requirements.txt') as fd:
+        return fd.read().splitlines()
+
+
+def gofedlib_find_packages():
+    packages = find_packages()
+    # data files are often placed outside of package dir, we have to add them
+    # manually
+    additional = [
+        'gofedlib.config',
+        'gofedlib.docs',
+        'gofedlib.distribution.clients.fakedata',
+        'gofedlib.distribution.data',
+        'gofedlib.go.apidiff.testdata',
+        'gofedlib.go.importpath.data',
+        'gofedlib.providers.data',
+        'gofedlib.schemas'
+    ]
+    return packages + additional
+
+setup(
+    name='gofedlib',
+    version='0.1.0a1',
+    packages=gofedlib_find_packages(),
+    scripts=['gofedlib-cli'],
+    install_requires=get_requirements(),
+    cmdclass={'install': GofedlibInstall},
+    package_data={
+        'gofedlib.config': ['lib.conf'],
+        'gofedlib.docs': ['proposal.md', 'providers.md'],
+        'gofedlib.distribution.clients.fakedata': ['data.json'],
+        'gofedlib.distribution.clients.pkgdb': ['fakedata.json'],
+        'gofedlib.distribution.data': ['ip2package_mapping.json'],
+        'gofedlib.go.apidiff.testdata': ['api1.json', 'api2.json'],
+        'gofedlib.go.importpath.data': ['known_prefixes.json', 'native_packages.json'],
+        'gofedlib.go.symbolsextractor': ['parseGo.go', 'parseGo'],
+        'gofedlib.logger': ['logger.yaml'],
+        'gofedlib.providers.data': ['ip2pp_mapping.json'],
+        'gofedlib.schemas': [
+            'distribution_packages.json',
+            'golang_native_imports.json',
+            'import_path_to_package_name.json',
+            'import_path_to_provider_prefix.json',
+            'spec_model.json'
+        ]
+    },
+    author='Jan Chaloupka',
+    author_email='[email protected]',
+    maintainer='Jan Chaloupka',
+    maintainer_email='[email protected]',
+    description='A set of python modules carrying operation related to Go source codes analysis used in Gofed',
+    url='https://github.com/gofed/lib',
+    license='GPL',
+    keywords='gofed golang API dependencies',
+)