go_mod.py

# Copyright (c) nexB Inc. and others. All rights reserved.
# ScanCode is a trademark of nexB Inc.
# SPDX-License-Identifier: Apache-2.0
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
# See https://github.com/nexB/scancode-toolkit for support or download.
# See https://aboutcode.org for more information about nexB OSS projects.
#

import io
import re

import attr
from packageurl import PackageURL


@attr.s()
class GoModule(object):
    namespace = attr.ib(default=None)
    name = attr.ib(default=None)
    version = attr.ib(default=None)
    module = attr.ib(default=None)
    require = attr.ib(default=None)
    exclude = attr.ib(default=None)
    local_replacements = attr.ib(default=None)

    def purl(self, include_version=True):
        version = None
        if include_version:
            version = self.version
        return PackageURL(
                    type='golang',
                    namespace=self.namespace,
                    name=self.name,
                    version=version
                ).to_string()


# Regex expressions to parse different types of go.mod file dependency
parse_module = re.compile(
    r'(?P<type>[^\s]+)'
    r'(\s)+'
    r'(?P<ns_name>[^\s]+)'
    r'\s?'
    r'(?P<version>(.*))'
).match

parse_dep_link = re.compile(
    r'.*?'
    r'(?P<ns_name>[^\s]+)'
    r'\s+'
    r'(?P<version>(.*))'
).match

parse_rep_link = re.compile(
    r"(?P<ns_name>\S+)"
    r"(?:\s+(?P<version>\S+))?"
    r"\s*=>\s*"
    r"(?P<replacement_ns_name>\S+)"
    r"(?:\s+(?P<replacement_version>\S+))?"
).match

def preprocess(line):
    """
    Return line string after removing commented portion and excess spaces.
    """
    if "//" in line:
        line = line[:line.index('//')]
    line = line.strip()
    return line

def parse_replace_directive(line):
    parsed_replace = parse_rep_link(line)
    ns_name = parsed_replace.group("ns_name")
    version = parsed_replace.group("version")
    namespace, _, name = ns_name.rpartition("/")
    original_module = {
        "namespace": namespace,
        "name": name,
        "version": version
    }

    replacement_ns_name = parsed_replace.group("replacement_ns_name")
    replacement_version = parsed_replace.group("replacement_version")
    is_local = replacement_ns_name.startswith("./") or replacement_ns_name.startswith("../")

    if is_local:
        replacement_namespace = None
        replacement_name = replacement_ns_name
    else:
        replacement_namespace, _, replacement_name = replacement_ns_name.rpartition("/")

    replacement_module = {
        "namespace": replacement_namespace,
        "name": replacement_name,
        "version": replacement_version,
        "is_local": is_local,
        "local_path": replacement_ns_name if is_local else None
    }

    return original_module, replacement_module

def handle_replace_directive(line, require, exclude, local_replacements):
    original, replacement = parse_replace_directive(line)
    exclude.append(
        GoModule(
            namespace=original.get('namespace'),
            name=original.get('name'),
            version=original.get('version'),
        )
    )

    if replacement.get('is_local'):
        local_replacements.append({
            'replaces': f"{original.get('namespace')}/{original.get('name')}",
            'local_path': replacement.get('local_path')
        })
    else:
        require.append(
            GoModule(
                namespace=replacement.get('namespace'),
                name=replacement.get('name'),
                version=replacement.get('version')
            )
        )

def parse_gomod(location):
    """
    Return a dictionary containing all the important go.mod file data.

    Handle go.mod files from Go.
    See https://golang.org/ref/mod#go.mod-files for details

    For example:

        module example.com/my/thing

        go 1.12

        require example.com/other/thing v1.0.2
        require example.com/new/thing v2.3.4
        exclude example.com/old/thing v1.2.3
        require (
            example.com/new/thing v2.3.4
            example.com/old/thing v1.2.3
        )
        require (
            example.com/new/thing v2.3.4
            example.com/old/thing v1.2.3
        )

    Each module line is in the form
        require github.com/davecgh/go-spew v1.1.1
    or
        exclude github.com/davecgh/go-spew v1.1.1
    or
        module github.com/alecthomas/participle

    For example::

        >>> p = parse_module('module github.com/alecthomas/participle')
        >>> assert p.group('type') == ('module')
        >>> assert p.group('ns_name') == ('github.com/alecthomas/participle')

        >>> p = parse_module('require github.com/davecgh/go-spew v1.1.1')
        >>> assert p.group('type') == ('require')
        >>> assert p.group('ns_name') == ('github.com/davecgh/go-spew')
        >>> assert p.group('version') == ('v1.1.1')

    A line for require or exclude can be in the form:

        github.com/davecgh/go-spew v1.1.1

    For example::

        >>> p = parse_dep_link('github.com/davecgh/go-spew v1.1.1')
        >>> assert p.group('ns_name') == ('github.com/davecgh/go-spew')
        >>> assert p.group('version') == ('v1.1.1')
    """
    with io.open(location, encoding='utf-8', closefd=True) as data:
        lines = data.readlines()

    gomods = GoModule()
    require = []
    exclude = []
    local_replacements = []

    for i, line in enumerate(lines):
        line = preprocess(line)

        if 'require' in line and '(' in line:
            for req in lines[i + 1:]:
                req = preprocess(req)
                if ')' in req:
                    break
                parsed_dep_link = parse_dep_link(req)
                ns_name = parsed_dep_link.group('ns_name')
                namespace, _, name = ns_name.rpartition('/')
                if parsed_dep_link:
                    require.append(GoModule(
                            namespace=namespace,
                            name=name,
                            version=parsed_dep_link.group('version')
                        )
                    )
            continue

        if 'exclude' in line and '(' in line:
            for exc in lines[i + 1:]:
                exc = preprocess(exc)
                if ')' in exc:
                    break
                parsed_dep_link = parse_dep_link(exc)
                ns_name = parsed_dep_link.group('ns_name')
                namespace, _, name = ns_name.rpartition('/')
                if parsed_dep_link:
                    exclude.append(GoModule(
                            namespace=namespace,
                            name=name,
                            version=parsed_dep_link.group('version')
                        )
                    )
            continue

        if 'replace' in line and '(' in line:
            for rep in lines[i + 1:]:
                rep = preprocess(rep)
                if ')' in rep:
                    break
                handle_replace_directive(rep, require, exclude, local_replacements)
            continue

        if 'replace' in line and '=>' in line:
            line = line.lstrip("replace").strip()
            handle_replace_directive(line, require, exclude, local_replacements)
            continue

        parsed_module_name = parse_module(line)
        if parsed_module_name:
            ns_name = parsed_module_name.group('ns_name')
            namespace, _, name = ns_name.rpartition('/')

        if 'module' in line:
            gomods.namespace = namespace
            gomods.name = name
            continue

        if 'require' in line:
            require.append(GoModule(
                    namespace=namespace,
                    name=name,
                    version=parsed_module_name.group('version')
                )
            )
            continue

        if 'exclude' in line:
            exclude.append(GoModule(
                    namespace=namespace,
                    name=name,
                    version=parsed_module_name.group('version')
                )
            )
            continue

    gomods.require = require
    gomods.exclude = exclude
    gomods.local_replacements = local_replacements

    return gomods


# Regex expressions to parse go.sum file dependency
# dep example: github.com/BurntSushi/toml v0.3.1 h1:WXkYY....
get_dependency = re.compile(
    r'(?P<ns_name>[^\s]+)'
    r'\s+'
    r'(?P<version>[^\s]+)'
    r'\s+'
    r'h1:(?P<checksum>[^\s]*)'
).match

def parse_gosum(location):
    """
    Return a list of GoSum from parsing the go.sum file at `location`.

    Handles go.sum file from Go.

    See https://blog.golang.org/using-go-modules for details

    A go.sum file contains pinned Go modules checksums of two styles:

    For example::

    github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
    github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=

    ... where the line with /go.mod is for a check of that go.mod file
    and the other line contains a dirhash for that path as documented as
    https://pkg.go.dev/golang.org/x/mod/sumdb/dirhash

    For example::

        >>> p = get_dependency('github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=')
        >>> assert p.group('ns_name') == ('github.com/BurntSushi/toml')
        >>> assert p.group('version') == ('v0.3.1')
        >>> assert p.group('checksum') == ('WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=')
    """
    with io.open(location, encoding='utf-8', closefd=True) as data:
        lines = data.readlines()

    gosums = []

    for line in lines:
        line = line.replace('/go.mod', '')
        parsed_dep = get_dependency(line)

        ns_name = parsed_dep.group('ns_name')
        namespace, _, name = ns_name.rpartition('/')

        dep = GoModule(
                namespace=namespace,
                name=name,
                version=parsed_dep.group('version')
            )

        if dep in gosums:
            continue

        gosums.append(dep)

    return gosums