From 9671ef53ae97fa8277e2521a77762773927f1e8c Mon Sep 17 00:00:00 2001 From: Tomasz Buchert Date: Wed, 5 Aug 2015 20:10:46 +0200 Subject: [PATCH 1/1] initial commit --- .gitignore | 5 + TODO.org | 12 + bundle-files/Dockerfile | 28 ++ bundle-files/steps/01-upgrade | 18 + bundle-files/steps/02-install-utils | 15 + bundle-files/steps/03-install-deps | 16 + bundle-files/steps/04-extract-source | 16 + bundle-files/steps/05-build | 27 ++ bundle-files/steps/build-tar | 17 + debian/README | 6 + debian/changelog | 5 + debian/compat | 1 + debian/control | 25 ++ debian/copyright | 32 ++ debian/debocker.manpages | 1 + debian/docs | 1 + debian/rules | 5 + debian/source/format | 1 + debocker | 579 +++++++++++++++++++++++++++ doc/Makefile | 15 + doc/debocker.8 | 199 +++++++++ doc/debocker.8.html | 259 ++++++++++++ doc/debocker.8.ronn | 164 ++++++++ doc/usecases-new.org | 8 + doc/usecases.txt | 89 ++++ setup.py | 51 +++ tests/Makefile | 10 + tests/test-inception.sh | 7 + utils/debian-build/Dockerfile | 22 + utils/push-new-image | 11 + 30 files changed, 1645 insertions(+) create mode 100644 .gitignore create mode 100644 TODO.org create mode 100644 bundle-files/Dockerfile create mode 100755 bundle-files/steps/01-upgrade create mode 100755 bundle-files/steps/02-install-utils create mode 100755 bundle-files/steps/03-install-deps create mode 100755 bundle-files/steps/04-extract-source create mode 100755 bundle-files/steps/05-build create mode 100755 bundle-files/steps/build-tar create mode 100644 debian/README create mode 100644 debian/changelog create mode 100644 debian/compat create mode 100644 debian/control create mode 100644 debian/copyright create mode 100644 debian/debocker.manpages create mode 100644 debian/docs create mode 100755 debian/rules create mode 100644 debian/source/format create mode 100755 debocker create mode 100644 doc/Makefile create mode 100644 doc/debocker.8 create mode 100644 doc/debocker.8.html create mode 100644 doc/debocker.8.ronn create mode 100644 doc/usecases-new.org create mode 100644 doc/usecases.txt create mode 100644 setup.py create mode 100644 tests/Makefile create mode 100755 tests/test-inception.sh create mode 100644 utils/debian-build/Dockerfile create mode 100755 utils/push-new-image diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..254aa1f --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +build/ +debocker.egg-info/ +dist/ +*.pyc + diff --git a/TODO.org b/TODO.org new file mode 100644 index 0000000..0978fa4 --- /dev/null +++ b/TODO.org @@ -0,0 +1,12 @@ +* Things to do (ordered by priority) +** add --debug to build the package with all debugging information +** add more tests +** replace --from with "docker rmi --no-prune=true" +** it's not clear if running the build process as root is safe; switch to a normal user maybe? +* Things to consider +** consider using docker API (python-docker) +* Done +** skip .git directory from sources; actually, probably just store orig tarball +** let pass build flags to dpkg-buildpackage +** the bundles are not reproducible => make them so +** add support for other dists (stable, experimental, etc.) diff --git a/bundle-files/Dockerfile b/bundle-files/Dockerfile new file mode 100644 index 0000000..f766bb7 --- /dev/null +++ b/bundle-files/Dockerfile @@ -0,0 +1,28 @@ +FROM ${image} +MAINTAINER Tomasz Buchert + +ENV DEBIAN_FRONTEND noninteractive + +# +# ADD ./steps /root/steps +# + +ADD ./steps/01-upgrade /root/steps/01-upgrade +RUN /root/steps/01-upgrade '${args_01}' + +ADD ./steps/02-install-utils /root/steps/02-install-utils +RUN /root/steps/02-install-utils '${args_02}' + +ADD ./control /root/control +ADD ./steps/03-install-deps /root/steps/03-install-deps +RUN /root/steps/03-install-deps '${args_03}' + +COPY ./source/* ./info /root/source/ +ADD ./steps/04-extract-source /root/steps/04-extract-source +RUN ./root/steps/04-extract-source '${args_04}' + +ADD ./steps/05-build /root/steps/05-build +ADD ./buildinfo /root/source/ +RUN /root/steps/05-build '${args_05}' + +ADD ./steps/build-tar /root/steps/build-tar diff --git a/bundle-files/steps/01-upgrade b/bundle-files/steps/01-upgrade new file mode 100755 index 0000000..ad3472d --- /dev/null +++ b/bundle-files/steps/01-upgrade @@ -0,0 +1,18 @@ +#!/bin/bash +# +# updates Debian image +# + +echo "== STAGE 01 (upgrading system) ==" + +set -eux + +# TODO: currently, we don't need deb-src +# echo "deb-src http://httpredir.debian.org/debian unstable main" \ +# > /etc/apt/sources.list.d/unstable-deb-src.list + +apt-get -y update +apt-get -y upgrade +apt-get -y clean + +echo "== ENDSTAGE ==" diff --git a/bundle-files/steps/02-install-utils b/bundle-files/steps/02-install-utils new file mode 100755 index 0000000..4bc6fba --- /dev/null +++ b/bundle-files/steps/02-install-utils @@ -0,0 +1,15 @@ +#!/bin/bash +# +# installs build tools +# + +echo "== STAGE 02 (installing build tools) ==" + +set -eux + +apt-get -y --no-install-recommends \ + install devscripts pbuilder build-essential aptitude lintian + +apt-get -y clean + +echo "== ENDSTAGE ==" diff --git a/bundle-files/steps/03-install-deps b/bundle-files/steps/03-install-deps new file mode 100755 index 0000000..1483ba5 --- /dev/null +++ b/bundle-files/steps/03-install-deps @@ -0,0 +1,16 @@ +#!/bin/bash +# +# installs package dependencies +# assumes that /root/control exists +# (presumably added by Dockerfile) + +echo "== STAGE 03 (install dependencies) ==" + +set -eux + +/usr/lib/pbuilder/pbuilder-satisfydepends-aptitude \ + --control /root/control + +apt-get -y clean + +echo "== ENDSTAGE ==" diff --git a/bundle-files/steps/04-extract-source b/bundle-files/steps/04-extract-source new file mode 100755 index 0000000..ccd5b98 --- /dev/null +++ b/bundle-files/steps/04-extract-source @@ -0,0 +1,16 @@ +#!/bin/bash +# +# extracts sources +# the files are in /root/source + +echo "== STAGE 04 (extract sources) ==" + +set -eux + +cd /root/source +dpkg-source -x *.dsc ./build + +# remove .dsc so it does not collide with the build +rm *.dsc + +echo "== ENDSTAGE ==" diff --git a/bundle-files/steps/05-build b/bundle-files/steps/05-build new file mode 100755 index 0000000..493cf31 --- /dev/null +++ b/bundle-files/steps/05-build @@ -0,0 +1,27 @@ +#!/bin/bash +# +# builds package +# + +echo "== STAGE 04 (building package) ==" + +set -eux + +source /root/source/buildinfo + +echo "Build flags: ${flags}" + +cd /root/source/build + +# TODO: also store stderr? +dpkg-buildpackage ${flags} | tee /root/source/build.log + +cd /root/source +name=$(basename --suffix .dsc *.dsc) +mv build.log ${name}.build + +echo "+++ lintian output +++" +lintian --pedantic --display-info *.changes +echo "+++ end of lintian output +++" + +echo "== ENDSTAGE ==" diff --git a/bundle-files/steps/build-tar b/bundle-files/steps/build-tar new file mode 100755 index 0000000..06fb0e0 --- /dev/null +++ b/bundle-files/steps/build-tar @@ -0,0 +1,17 @@ +#!/bin/bash +# takes all build files and extracts them as tar +# on the stdout + +set -eu + +source /root/source/info + +cd /root/source/ + +if [ "${format}" = "native" ]; then + # native + exec tar -cf - *.build *.changes *.deb *.dsc *.tar.* +else + # non-native + exec tar -cf - *.build *.changes *.deb *.dsc *.debian.tar.* *.orig.tar.* +fi diff --git a/debian/README b/debian/README new file mode 100644 index 0000000..16cd7b8 --- /dev/null +++ b/debian/README @@ -0,0 +1,6 @@ +The Debian Package debocker +---------------------------- + +Comments regarding the Package + + -- Tomasz Buchert Mon, 20 Jul 2015 10:43:06 +0200 diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..0dd4dd2 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +debocker (0.2) unstable; urgency=low + + * Initial Release. + + -- Tomasz Buchert Mon, 20 Jul 2015 10:43:06 +0200 diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000..ec63514 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +9 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..fa4cd06 --- /dev/null +++ b/debian/control @@ -0,0 +1,25 @@ +Source: debocker +Section: devel +Priority: extra +Maintainer: Tomasz Buchert +Build-Depends: debhelper (>= 9), + dh-python, + python3 (>= 3.3), + python3-setuptools +Standards-Version: 3.9.6 +Homepage: https://people.debian.org/~tomasz/debocker.html +Vcs-Git: git://anonscm.debian.org/collab-maint/debocker.git +Vcs-Browser: https://anonscm.debian.org/cgit/collab-maint/debocker.git + +Package: debocker +Architecture: all +Depends: docker.io, + python3 (>= 3.3), + python3-click, + ${misc:Depends}, + ${shlibs:Depends}, + ${python3:Depends} +Description: docker-powered package builder for Debian + debocker builds Debian packages using docker. It is also capable to + create bundles that can be shared to build the same package on + different nodes. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..9165def --- /dev/null +++ b/debian/copyright @@ -0,0 +1,32 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: debocker +Source: https://people.debian.org/~tomasz/debocker.html + +Files: * +Copyright: 2015 Tomasz Buchert +License: GPL-3.0+ + +Files: debian/* +Copyright: 2015 Tomasz Buchert +License: GPL-3.0+ + +Files: setup.py +Copyright: 2015 Dariusz Dwornikowski +License: GPL-3.0+ + +License: GPL-3.0+ + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + . + This package is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + . + You should have received a copy of the GNU General Public License + along with this program. If not, see . + . + On Debian systems, the complete text of the GNU General + Public License version 3 can be found in "/usr/share/common-licenses/GPL-3". diff --git a/debian/debocker.manpages b/debian/debocker.manpages new file mode 100644 index 0000000..5b7ce37 --- /dev/null +++ b/debian/debocker.manpages @@ -0,0 +1 @@ +doc/debocker.8 diff --git a/debian/docs b/debian/docs new file mode 100644 index 0000000..0bf55fd --- /dev/null +++ b/debian/docs @@ -0,0 +1 @@ +TODO.org diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..2a54fce --- /dev/null +++ b/debian/rules @@ -0,0 +1,5 @@ +#!/usr/bin/make -f +#export DH_VERBOSE = 1 + +%: + dh $@ --buildsystem=pybuild --with=python3 diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..89ae9db --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (native) diff --git a/debocker b/debocker new file mode 100755 index 0000000..480a03b --- /dev/null +++ b/debocker @@ -0,0 +1,579 @@ +#!/usr/bin/env python3 +# coding: utf-8 + +import click +import re +import os +import hashlib +from tempfile import TemporaryDirectory +from os.path import isdir, join, isfile, abspath, dirname, \ + realpath, splitext, relpath +from subprocess import check_call, check_output, CalledProcessError, DEVNULL +from shutil import which, copytree, copyfile +from shlex import quote as shell_quote +from collections import OrderedDict +from string import Template +from datetime import datetime + +__version__ = "0.2" + +NONE, LOW = 0, 1 +VERBOSITY = NONE + +# USEFUL ROUTINES + +def fail(msg, *params): + raise click.ClickException(msg.format(*params)) + +def cached_constant(f): + cache = [] + def _f(): + if len(cache) == 0: + cache.append(f()) + return cache[0] + return _f + +def cached_property(f): + key = '_cached_' + f.__name__ + def _f(self): + if not hasattr(self, key): + setattr(self, key, f(self)) + return getattr(self, key) + return property(_f) + +def Counter(v = 0): + v = [ v ] + def _f(): + v[0] += 1 + return v[0] + return _f + +tmpdir = cached_constant(lambda: TemporaryDirectory()) +tmpunique = Counter() +CURRENT_TIME = datetime.utcnow().isoformat() + +def tmppath(name = None, directory = False): + '''get a temp. path''' + count = tmpunique() + tmp_directory = tmpdir().name + if name is None: + name = 'tmp' + tmp_path = join(tmp_directory, '{:03d}-{}'.format(count, name)) + if directory: + os.mkdir(tmp_path) + return tmp_path + +def log(msg, v = 0): + if VERBOSITY == NONE: + if v == 0: + click.secho('LOG {}'.format(msg), fg = 'green') + else: + if v <= VERBOSITY: + click.secho('LOG[{}] {}'.format(v, msg), fg = 'green') + +def log_check_call(cmd, **kwds): + log('Run (check): {}'.format(cmd), LOW) + return check_call(cmd, **kwds) + +def log_check_output(cmd, **kwds): + log('Run (output): {}'.format(cmd), LOW) + return check_output(cmd, **kwds) + +@cached_constant +def find_bundle_files(): + '''finds the location of bundle files''' + debocker_dir = dirname(abspath(realpath(__file__))) + locations = [ debocker_dir, '/usr/share/debocker' ] + for loc in locations: + bundle_files = join(loc, 'bundle-files') + if isdir(bundle_files): + log("Bundle files found in '{}'.".format(bundle_files), LOW) + return bundle_files + fail('could not find bundle files') + +def assert_command(cmd): + if which(cmd) is None: + fail("command '{}' is not available", cmd) + +@cached_constant +def assert_docker(): + if which('docker') is None: + fail('docker is not available') + +def extract_pristine_tar(path, candidates): + if which('pristine-tar') is None: + log('No pristine-tar available.', LOW) + return None + try: + tars = log_check_output([ 'pristine-tar', 'list' ], cwd = path) + except CalledProcessError: + log('Error in pristine-tar - giving up.', LOW) + return None + # let's hope that no non-utf8 files are stored + thelist = tars.decode('utf-8').split() + log('Pristine tarballs: {}'.format(thelist), LOW) + matches = list(set(thelist) & set(candidates)) + if len(matches) > 0: + m = matches[0] + log("Found a match: '{}'.".format(m), LOW) + log_check_call([ 'pristine-tar', 'checkout', m ], + cwd = path, stderr = DEVNULL) + try: + src = join(path, m) + dst = tmppath(m) + copyfile(src, dst) + log("Tarball extracted to '{}'.".format(dst), LOW) + finally: + os.unlink(src) + return dst + else: + log('No matching pristine tar found.', LOW) + return None + +class Package: + + def __init__(self, path): + self.path = path + self.debian = join(self.path, 'debian') + self.control = join(self.debian, 'control') + self.changelog = join(self.debian, 'changelog') + self.source_format = join(self.debian, 'source', 'format') + + def is_valid(self): + '''verifies that the current directory is a debian package''' + return isdir(self.debian) and isfile(self.control) and \ + isfile(self.source_format) and self.format in ['native', 'quilt'] + + def assert_is_valid(self): + if not self.is_valid(): + fail('not in debian package directory') + + @cached_property + def format(self): + with open(self.source_format) as f: + line = f.readline().strip() + m = re.match(r'^3\.0 \((native|quilt)\)', line) + if not m: + fail('unsupported format ({})', line) + fmt = m.group(1) + log("Detected format '{}'.".format(fmt), LOW) + return fmt + + @cached_property + def native(self): + return self.format == 'native' + + @cached_property + def name(self): + with open(self.control) as f: + for line in f: + m = re.match(r'^Source: (\S+)$', line) + if m: + return m.group(1) + fail('could not find the name of the package') + + @cached_property + def long_version(self): + '''long version''' + with open(self.changelog) as f: + line = f.readline() + m = re.match(r'^(\S+) \((\S+)\)', line) + if not m: + fail("could not parse package version (from '{}')", line.strip()) + return m.group(2) + + @cached_property + def version(self): + '''upstream version''' + if self.native: + return self.long_version + else: + m = re.match(r'^(.+)-(\d+)$', self.long_version) + if not m: + fail('could not parse version ({})', self.long_version) + return m.group(1) + + @cached_property + def orig_tarball_candidates(self): + '''possible original upstream tarballs''' + formats = [ 'xz', 'gz', 'bz2' ] + names = [ '{}_{}.orig.tar.{}'.format(self.name, self.version, fmt) + for fmt in formats ] + return names + + @cached_property + def orig_tarball(self): + '''finds the original tarball''' + for name in self.orig_tarball_candidates: + tarball = join(self.path, '..', name) + log("Trying tarball candidate '{}'.".format(tarball), LOW) + if isfile(tarball): + log("Original tarball found at '{}'.".format(tarball), LOW) + return tarball + result = extract_pristine_tar(self.path, self.orig_tarball_candidates) + if result is not None: + return result + fail('could not find original tarball') + + def assert_orig_tarball(self): + if self.native: + # for now, we just tar the current directory + path = tmppath('{}_{}.tar.xz'.format( + self.name, self.version)) + with open(path, 'wb') as output: + tar = [ 'tar', 'c', '--xz', '--exclude=.git', + '-C', self.path, '.' ] + log_check_call(tar, stdout = output) + return path + else: + return self.orig_tarball # simple alias + + def tar_package_debian(self, output, comp = None): + # TODO: make it reproducible + compressions = { + 'xz': '--xz' + } + tar = [ 'tar', 'c' ] + if comp: + tar += [ compressions[comp] ] + debian = join(self.path, 'debian') + filelist = get_reproducible_filelist(debian, self.path) + tar += [ '--no-recursion', '-C', self.path ] + tar += filelist + log_check_call(tar, stdout = output) + + def tar_original_tarball(self, output, stdout): + orig = self.assert_orig_tarball() + return orig # for now + + def build_docker_tarball_to_fd(self, output, buildinfo): + '''builds the docker tarball that builds the package''' + controlfile = join(self.path, 'debian', 'control') + controlfile = abspath(controlfile) + debianfile = tmppath('debian.tar.xz') + with open(debianfile, 'wb') as debian: + self.tar_package_debian(debian, comp = 'xz') + originalfile = self.assert_orig_tarball() + originalfile = abspath(originalfile) + if self.native: + make_native_bundle(self.name, self.version, + controlfile, originalfile, + buildinfo, output) + else: + make_quilt_bundle(self.name, self.long_version, + controlfile, originalfile, debianfile, + buildinfo, output) + + def build_docker_tarball(self, filename, buildinfo): + with open(filename, 'wb') as output: + self.build_docker_tarball_to_fd(output, buildinfo) + +def calculate_md5_and_size(path): + md5 = hashlib.md5() + count = 0 + with open(path, 'rb') as f: + while True: + buff = f.read(8192) + if len(buff) == 0: + break + count += len(buff) + md5.update(buff) + return md5.hexdigest(), count + +def make_native_bundle(name, version, control, + source, buildinfo, output): + dsc_name = '{}_{}.dsc'.format(name, version) + bundler = Bundler(name, version, control, dsc_name, + buildinfo, native = True) + _, ext = splitext(source) + source_name = '{}_{}.tar{}'.format(name, version, ext) + bundler.add_source_file(source_name, source, 'source_tarball') + bundler.write_bundle(output = output) + +def make_quilt_bundle(name, version, control, + original, debian, buildinfo, output): + dsc_name = '{}_{}.dsc'.format(name, version) + bundler = Bundler(name, version, control, dsc_name, + buildinfo, native = False) + _, oext = splitext(original) + _, dext = splitext(debian) + # TODO: improve + uversion = version.split('-')[0] + original_name = '{}_{}.orig.tar{}'.format(name, uversion, oext) + debian_name = '{}_{}.debian.tar{}'.format(name, version, dext) + bundler.add_source_file(original_name, original, 'original_tarball') + bundler.add_source_file(debian_name, debian, 'debian_tarball') + bundler.write_bundle(output = output) + +def get_reproducible_filelist(path, base = None): + if base is None: + base = path + elements = [] + for p, ds, fs in os.walk(path): + p = relpath(p, base) + if p != '.': + p = join('.', p) + elements += [ join(p, x) for x in ds ] + elements += [ join(p, x) for x in fs ] + return sorted(elements) + + +# STEPS + +STEPS = OrderedDict([ + ('end', None), + ('build', 'args_05'), + ('extract-source', 'args_04'), + ('install-deps', 'args_03'), + ('install-utils', 'args_02'), + ('upgrade', 'args_01') +]) + +class Bundler: + + def __init__(self, name, version, control, dsc_name, buildinfo, native): + self.name = name + self.version = version + self.native = native + self.control = control + self.sources = [] + self.dsc_name = dsc_name + self.step_name = buildinfo['step'] + self.step = STEPS[self.step_name] + self.image = buildinfo['image'] + self.buildinfo = buildinfo + self.wdir = tmppath('bundle', directory = True) + + @property + def format_string(self): + return ('3.0 (native)' if self.native else '3.0 (quilt)') + + def add_source_file(self, name, path, tag): + md5, size = calculate_md5_and_size(path) + self.sources.append({ + 'name': name, + 'path': path, + 'md5': md5, + 'size': size, + 'tag': tag + }) + + def write_dsc_file(self): + '''makes minimal, yet functional .dsc file''' + path = join(self.wdir, 'source', self.dsc_name) + with open(path, 'w') as f: + f.write('Format: {}\n'.format(self.format_string)) + f.write('Source: {}\n'.format(self.name)) + f.write('Version: {}\n'.format(self.version)) + f.write('Files:\n') + for s in self.sources: + f.write(' {} {} {}\n'.format(s['md5'], s['size'], s['name'])) + + def write_info_file(self, info): + path = join(self.wdir, 'info') + with open(path, 'w') as f: + for k, v in info.items(): + f.write("{}={}\n".format(k, v)) + + def write_buildinfo_file(self): + path = join(self.wdir, 'buildinfo') + with open(path, 'w') as f: + f.write("flags='{}'\n".format(self.buildinfo['flags'])) + + def write_bundle(self, output): + '''writes bundle to a given file''' + + os.makedirs(join(self.wdir, 'source')) + + info = OrderedDict() + info['bundle_version'] = __version__ + info['name'] = self.name + info['version'] = self.version + info['format'] = ('native' if self.native else 'quilt') + + def make_link(target, parts): + return os.symlink(target, join(self.wdir, *parts)) + + # control file + make_link(self.control, [ 'control' ]) + + # dsc file + self.write_dsc_file() + info['dsc_name'] = self.dsc_name + + # sources + for s in self.sources: + name = s['name'] + tag = s['tag'] + make_link(s['path'], [ 'source', name ]) + info[tag] = name + + # info & buildinfo + self.write_info_file(info) + self.write_buildinfo_file() + + # bundle files + bundle_files = find_bundle_files() + + copytree(join(bundle_files, 'steps'), + join(self.wdir, 'steps')) + dockertmpl = join(bundle_files, 'Dockerfile') + with open(dockertmpl, 'r') as df: + t = Template(df.read()) + ctx = { + 'image': self.image, + 'args_01': 'none', + 'args_02': 'none', + 'args_03': 'none', + 'args_04': 'none', + 'args_05': 'none' + } + if self.step: + log("Replacing '{}' with '{}'.".format( + self.step, CURRENT_TIME)) + if self.step not in ctx: + fail('internal error in dockerfile template') + ctx[self.step] = CURRENT_TIME + rendered = t.substitute(ctx) + dockerfile = join(self.wdir, 'Dockerfile') + with open(dockerfile, 'w') as f: + f.write(rendered) + + file_list = get_reproducible_filelist(self.wdir) + # tar everything + tar = [ 'tar', 'c', '-h', '--numeric-owner' ] + tar += [ '--no-recursion' ] + tar += [ '--owner=0', '--group=0' ] + tar += [ '--mtime=1970-01-01' ] + tar += [ '-C', self.wdir ] + tar += file_list + log_check_call(tar, stdout = output) + +def docker_build_bundle(bundle, no_cache, pull): + '''builds the given image and returns the final image''' + # TODO: quite ugly, cannot be done cleaner? + build_log = tmppath() + bundle_esc = shell_quote(bundle) + build_log_esc = shell_quote(build_log) + docker_opts = [] + if no_cache: + docker_opts.append('--no-cache') + if pull: + docker_opts.append('--pull') + docker_opts = ' '.join(docker_opts) + log_check_call('docker build {} - < {} | tee {}'.format( + docker_opts, bundle_esc, build_log_esc), shell = True) + with open(build_log) as f: + s = f.read().strip() + ms = re.findall(r'Successfully built (\S+)', s) + if len(ms) != 1: + fail('cannot parse logs (build failed?)') + image = ms[0] + return image + +# CLI INTERFACE + +@click.group() +@click.option('-v', '--verbose', count=True, + help = 'be verbose, repeat for more effect') +def cli(verbose): + global VERBOSITY + VERBOSITY = verbose + +@cli.command(help = 'Write tar bundle') +@click.argument('path', default = '.') +@click.option('-o', '--output', default = None, metavar = 'FILE', + help = 'output file') +@click.option('-f', '--flags', default = '', metavar = 'FLAGS', + help = 'build flags') +@click.option('step', '--from', default = 'end', + help = 'start from the given step', + type = click.Choice(STEPS.keys())) +@click.option('--image', default = 'debian:unstable', metavar = 'IMAGE', + help = 'base docker image') +def bundle(path, output, flags, step, image): + pkg = Package(path) + pkg.assert_is_valid() + if output is None: + name = '{}_{}_bundle.tar'.format(pkg.name, pkg.long_version) + output = join('..', name) + log('Preparing bundle for {} ({})...'.format(pkg.name, pkg.version)) + if not pkg.native: + pkg.assert_orig_tarball() + pkg.build_docker_tarball(output, { + 'flags': flags, 'step': step, 'image': image }) + log("Bundle created in '{}'.".format(output)) + if VERBOSITY > NONE: + md5, size = calculate_md5_and_size(output) + log('Bundle hash and size: {}, {}.'.format(md5, size), LOW) + +@cli.command('build-bundle', help = 'Build bundle') +@click.argument('bundle') +@click.option('-o', '--output', default = '.', metavar = 'DIRECTORY', + help = 'output directory') +@click.option('--sign', '-s', default = False, + is_flag = True, help = 'sign built package with debsign') +@click.option('no_cache', '--no-cache', default = False, + is_flag = True, help = 'do not use docker image cache') +@click.option('--pull', default = False, + is_flag = True, help = 'pull the newest base image') +def build_bundle(bundle, output, sign, no_cache, pull): + assert_docker() + if sign: + assert_command('debsign') + image = docker_build_bundle(bundle, + no_cache = no_cache, pull = pull) + log('Build successful (in {})'.format(image)) + # extract the build + build_tar = tmppath('build.tar') + with open(build_tar, 'wb') as f: + log_check_call([ 'docker', 'run', '--rm=true', + image, '/root/steps/build-tar' ], stdout = f) + log("Build tar stored in '{}'".format(build_tar)) + tar_list = log_check_output([ 'tar', 'tf', build_tar ]) + tar_files = tar_list.decode('utf-8').split() + log("Build files: {}".format(' '.join(tar_files)), LOW) + log_check_call([ 'tar', 'xf', build_tar, '-C', output ]) + log("Build files stored in '{}'.".format(output)) + if sign: + # TODO: needs devscripts, and hence will not work outside Debian + # we probably have to copy/fork debsign, cause signing within + # the container is not a good idea security-wise + changes = [ fn for fn in tar_files if fn.endswith('.changes') ] + if len(changes) != 1: + fail('could not find the changes files') + changes_path = join(output, changes[0]) + log("Trying to sign '{}'.".format(changes_path), LOW) + log_check_call([ 'debsign', changes_path ]) + +@cli.command(help = 'Build package') +@click.argument('path', default = '.') +@click.option('-o', '--output', default = '..', metavar = 'DIRECTORY', + help = 'output directory') +@click.option('--sign', '-s', default = False, + is_flag = True, help = 'sign built package with debsign') +@click.option('-f', '--flags', default = '', metavar = 'FLAGS', + help = 'build flags') +@click.option('no_cache', '--no-cache', default = False, + is_flag = True, help = 'do not use docker image cache') +@click.option('step', '--from', default = 'end', + help = 'start from the given step', + type = click.Choice(STEPS.keys())) +@click.option('--image', default = 'debian:unstable', metavar = 'IMAGE', + help = 'base docker image') +@click.option('--pull', default = False, + is_flag = True, help = 'pull the newest base image') +@click.pass_context +def build(ctx, path, output, sign, flags, no_cache, step, image, pull): + assert_docker() + if sign: + assert_command('debsign') + tarball_path = tmppath('bundle.tar') + ctx.invoke(bundle, path = path, output = tarball_path, + flags = flags, step = step, image = image) + ctx.invoke(build_bundle, + bundle = tarball_path, sign = sign, output = output, + no_cache = no_cache, pull = pull) + +if __name__ == '__main__': + cli.main() diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 0000000..33c8e51 --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,15 @@ +all: debocker.8 debocker.8.html + +debocker.8.html: debocker.8.ronn + ronn --html debocker.8.ronn --manual=debocker --organization=Debian + +debocker.8: debocker.8.ronn + ronn --roff debocker.8.ronn --manual=debocker --organization=Debian + +push-html: debocker.8.html + scp debocker.8.html debocker-server:debocker/index.html + +clean: + rm -f debocker.8 debocker.8.html + +.PHONY: push-html diff --git a/doc/debocker.8 b/doc/debocker.8 new file mode 100644 index 0000000..707219d --- /dev/null +++ b/doc/debocker.8 @@ -0,0 +1,199 @@ +.\" generated with Ronn/v0.7.3 +.\" http://github.com/rtomayko/ronn/tree/0.7.3 +. +.TH "DEBOCKER" "8" "July 2015" "Debian" "debocker" +. +.SH "NAME" +\fBdebocker\fR \- build Debian packages with docker +. +.SH "SYNOPSIS" +\fBdebocker\fR [\fIOPTS\fR] \fBCOMMAND\fR [\fICOMMAND OPTS\fR] [ARGS] +. +.SH "DESCRIPTION" +\fBDebocker\fR builds Debian packages inside docker\. The build process is contained in docker images and (almost) no other tools are needed to develop Debian packages on the main system\. Moreover, docker\'s image cache reuses the same system state whenever possible\. In particular, when a package is built for the second time, its dependecies and buildchain are not downloaded nor installed again\. +. +.P +\fBDebocker\fR is also able to create a self\-contained \fIbundle\fR with everything necessary to build a package with docker only\. +. +.P +You do not have to be root to run \fBdebocker\fR, but you have to be able to use docker(1) command\. In Debian, it means that you must be a member of the \fIdocker\fR group\. +. +.P +The build process consists of 5 steps: +. +.IP "1." 4 +\fIuprade\fR \- the base image is updated to the most recent packages (with apt\-get) +. +.IP "2." 4 +\fIinstall\-utils\fR \- the Debian toolchain is installed +. +.IP "3." 4 +\fIinstall\-deps\fR \- the build dependencies of the package are installed +. +.IP "4." 4 +\fIextract\-source\fR \- the source package is extracted +. +.IP "5." 4 +\fIbuild\fR \- the proper build is executed +. +.IP "" 0 +. +.SH "COMMANDS" +Each command accepts \fB\-\-help\fR option that shows its basic CLI usage\. +. +.IP "\(bu" 4 +\fBbundle\fR [\fBOPTS\fR] [\fBPATH\fR]: Create a tarball file containing sources of a package in the current directory, and a series of scripts to build it using docker\. The resulting \fIbundle\fR is stored in the parent directory as a tarball\. If the package is non\-native, the original tarball must be present in the parent directory\. However, if the original tarball is not present, debocker will try to extract it using pristine\-tar(1)\. +. +.IP +The bundle is independent from debocker and can be used with docker only (see \fIEXAMPLES\fR)\. +. +.IP +Arguments: +. +.IP +\fBPATH\fR: optional path to the package; defaults to the current directory +. +.IP +Options: +. +.IP +\fB\-o FILE\fR, \fB\-\-output FILE\fR: store bundle in \fBFILE\fR; the file is a traditional tarball with a Debian source package, a Dockerfile, and some utils +. +.IP +\fB\-f FLAGS\fR, \fB\-\-flags FLAGS\fR: define builds flags that bundle will use; these are passed to dpkg\-buildpackage +. +.IP +\fB\-\-from STEP\fR: invalidate \fBSTEP\fR causing docker to restart from this step, even if previous cache exists; possible values are: \fIbuild\fR, \fIextract\-source\fR, \fIinstall\-deps\fR, \fIinstall\-utils\fR, \fIupgrade\fR +. +.IP +\fB\-\-image IMAGE\fR: define the base docker image to use; defaults to \fIdebian:unstable\fR +. +.IP "\(bu" 4 +\fBbuild\-bundle\fR [\fBOPTS\fR] \fBBUNDLE\fR: Build a tarball file created with \fBbundle\fR by running the process in docker and extracting the final files to the current directory\. This multi\-step process takes advantage of docker\'s cache whenever possible, saving space and making successive builds very fast\. +. +.IP +Arguments: +. +.IP +\fBBUNDLE\fR: the location of a bundle to build +. +.IP +Options: +. +.IP +\fB\-o DIRECTORY\fR, \fB\-\-output DIRECTORY\fR: store the built files in \fBDIRECTORY\fR +. +.IP +\fB\-s\fR, \fB\-\-sign\fR: sign the results of the build; this requires installed \fBdebsign\fR (see devscripts(1)) +. +.IP +\fB\-\-no\-cache\fR: do not use docker\'s image cache (passed directly to \fIdocker build\fR); this effectively restarts the whole build from the start +. +.IP +\fB\-\-pull\fR: pull the newest base image if available (passed directly to \fIdocker build\fR) +. +.IP "\(bu" 4 +\fBbuild\fR [\fBOPTS\fR] [\fBPATH\fR]: Create a temporary bundle with \fBbundle\fR and build it with \fBbuild\-bundle\fR\. The respective options are passed unchanged to the subcommands (e\.g\., \fB\-s\fR can be used to sign packages)\. +. +.IP "" 0 +. +.SH "OPTIONS" +. +.TP +\fB\-v\fR, \fB\-\-verbose\fR +Make debocker\'s output verbose\. +. +.TP +\fB\-\-help\fR +Show summary of CLI usage\. +. +.P +Global options must be given before the name of the command\. +. +.SH "FILES" +There are no config files\. +. +.SH "EXAMPLES" +Assuming that you are in a directory with a Debian source package: +. +.IP "" 4 +. +.nf + +$ debocker build +. +.fi +. +.IP "" 0 +. +.P +will build the package in Debian unstable (the built files will be stored in the parent directory)\. If the build was successful, every subsequent run should use cache instead\. You may force rebuild at any stage by using \fB\-\-from\fR option\. To rebuild the package, you may use: +. +.IP "" 4 +. +.nf + +$ debocker build \-\-from=build +. +.fi +. +.IP "" 0 +. +.P +The \fBbuild\fR command is equivalent, save for the undeleted, intermediary file, with: +. +.IP "" 4 +. +.nf + +$ debocker bundle \-\-output /tmp/bundle\.tar +$ debocker build\-bundle /tmp/bundle\.tar \-\-output \.\. +. +.fi +. +.IP "" 0 +. +.P +You may pass custom flags to your build: +. +.IP "" 4 +. +.nf + +$ debocker build \-\-flags=\'\-j4\' +. +.fi +. +.IP "" 0 +. +.P +To create a (pseudo)\-reproducible, compressed bundle with your package and to build it using docker: +. +.IP "" 4 +. +.nf + +$ debocker bundle \-\-output /tmp/bundle\.tar +$ docker run \- < /tmp/bundle\.tar +. +.fi +. +.IP "" 0 +. +.SH "BUGS" +Debocker does not clean after itself\. If you are not careful, docker images may consume a lot of space\. +. +.P +And probably many more\. +. +.SH "AUTHOR" +Initial idea and coding has been done by Tomasz Buchert \fItomasz@debian\.org\fR\. +. +.P +Initial packaging, many ideas and a lot of support by Dariusz Dwornikowski\. +. +.P +The semi\-official homepage is \fIhttp://debocker\.debian\.net\fR\. +. +.SH "SEE ALSO" +pbuiler(8), docker(1), devscripts(1), pristine\-tar(1) diff --git a/doc/debocker.8.html b/doc/debocker.8.html new file mode 100644 index 0000000..7c8bcc4 --- /dev/null +++ b/doc/debocker.8.html @@ -0,0 +1,259 @@ + + + + + + debocker(8) - build Debian packages with docker + + + + +
+ + + +
    +
  1. debocker(8)
    2. +
  2. debocker
    3. +
  3. debocker(8)
    4. +
+ +

NAME

+

+ debocker - build Debian packages with docker +

+ +

SYNOPSIS

+ +

debocker [OPTS] COMMAND [COMMAND OPTS] [ARGS]

+ +

DESCRIPTION

+ +

Debocker builds Debian packages inside docker. The build process +is contained in docker images and (almost) no other tools are needed +to develop Debian packages on the main system. Moreover, docker's +image cache reuses the same system state whenever possible. In +particular, when a package is built for the second time, its +dependecies and buildchain are not downloaded nor installed again.

+ +

Debocker is also able to create a self-contained bundle with +everything necessary to build a package with docker only.

+ +

You do not have to be root to run debocker, but you have to be +able to use docker(1) command. In Debian, it means that you must be a +member of the docker group.

+ +

The build process consists of 5 steps:

+ +
    +
  1. uprade - the base image is updated to the most recent + packages (with apt-get)
    2. +
  2. install-utils - the Debian toolchain is installed
    3. +
  3. install-deps - the build dependencies of the package are + installed
    4. +
  4. extract-source - the source package is extracted
    5. +
  5. build - the proper build is executed
    6. +
+ + +

COMMANDS

+ +

Each command accepts --help option that shows its basic CLI usage.

+ +
    +

  • bundle [OPTS] [PATH]: Create a tarball file containing +sources of a package in the current directory, and a series of +scripts to build it using docker. The resulting bundle is stored +in the parent directory as a tarball. If the package is +non-native, the original tarball must be present in the parent +directory. However, if the original tarball is not present, +debocker will try to extract it using pristine-tar(1).

    + +

    The bundle is independent from debocker and can be used with +docker only (see EXAMPLES).

    + +

    Arguments:

    + +

    PATH: optional path to the package; defaults to the current +directory

    + +

    Options:

    + +

    -o FILE, --output FILE: store bundle in FILE; the file is a +traditional tarball with a Debian source package, a Dockerfile, +and some utils

    + +

    -f FLAGS, --flags FLAGS: define builds flags that bundle will +use; these are passed to dpkg-buildpackage

    + +

    --from STEP: invalidate STEP causing docker to restart from +this step, even if previous cache exists; possible values are: +build, extract-source, install-deps, install-utils, +upgrade

    + +

    --image IMAGE: define the base docker image to use; defaults to +debian:unstable

    • +

  • build-bundle [OPTS] BUNDLE: Build a tarball file created +with bundle by running the process in docker and extracting the +final files to the current directory. This multi-step process +takes advantage of docker's cache whenever possible, saving space +and making successive builds very fast.

    + +

    Arguments:

    + +

    BUNDLE: the location of a bundle to build

    + +

    Options:

    + +

    -o DIRECTORY, --output DIRECTORY: store the built files in +DIRECTORY

    + +

    -s, --sign: sign the results of the build; this requires +installed debsign (see devscripts(1))

    + +

    --no-cache: do not use docker's image cache (passed directly to +docker build); this effectively restarts the whole build from +the start

    + +

    --pull: pull the newest base image if available (passed directly +to docker build)

    • +

  • build [OPTS] [PATH]: Create a temporary bundle with bundle +and build it with build-bundle. The respective options are +passed unchanged to the subcommands (e.g., -s can be used to +sign packages).

    • +
+ + +

OPTIONS

+ +
+
-v, --verbose

Make debocker's output verbose.

+
--help

Show summary of CLI usage.

+
+ + +

Global options must be given before the name of the command.

+ +

FILES

+ +

There are no config files.

+ +

EXAMPLES

+ +

Assuming that you are in a directory with a Debian source package:

+ +
$ debocker build
+
+ +

will build the package in Debian unstable (the built files will be +stored in the parent directory). If the build was successful, every +subsequent run should use cache instead. You may force rebuild +at any stage by using --from option. To rebuild the package, +you may use:

+ +
$ debocker build --from=build
+
+ +

The build command is equivalent, save for the undeleted, +intermediary file, with:

+ +
$ debocker bundle --output /tmp/bundle.tar
+$ debocker build-bundle /tmp/bundle.tar --output ..
+
+ +

You may pass custom flags to your build:

+ +
$ debocker build --flags='-j4'
+
+ +

To create a (pseudo)-reproducible, compressed bundle with your package +and to build it using docker:

+ +
$ debocker bundle --output /tmp/bundle.tar
+$ docker run - < /tmp/bundle.tar
+
+ +

BUGS

+ +

Debocker does not clean after itself. If you are not careful, docker +images may consume a lot of space.

+ +

And probably many more.

+ +

AUTHOR

+ +

Initial idea and coding has been done by Tomasz Buchert +tomasz@debian.org.

+ +

Initial packaging, many ideas and a lot of support by Dariusz +Dwornikowski.

+ +

The semi-official homepage is http://debocker.debian.net.

+ +

SEE ALSO

+ +

pbuiler(8), docker(1), devscripts(1), pristine-tar(1)

+ + +
    +
  1. Debian
    2. +
  2. July 2015
    3. +
  3. debocker(8)
    4. +
+ +
+ + diff --git a/doc/debocker.8.ronn b/doc/debocker.8.ronn new file mode 100644 index 0000000..215bd05 --- /dev/null +++ b/doc/debocker.8.ronn @@ -0,0 +1,164 @@ +debocker(8) -- build Debian packages with docker +============================================= + +## SYNOPSIS + +`debocker` [] `COMMAND` [] [ARGS] + +## DESCRIPTION + +**Debocker** builds Debian packages inside docker. The build process +is contained in docker images and (almost) no other tools are needed +to develop Debian packages on the main system. Moreover, docker's +image cache reuses the same system state whenever possible. In +particular, when a package is built for the second time, its +dependecies and buildchain are not downloaded nor installed again. + +**Debocker** is also able to create a self-contained *bundle* with +everything necessary to build a package with docker only. + +You do not have to be root to run **debocker**, but you have to be +able to use docker(1) command. In Debian, it means that you must be a +member of the *docker* group. + +The build process consists of 5 steps: + + 1. *uprade* - the base image is updated to the most recent + packages (with apt-get) + 2. *install-utils* - the Debian toolchain is installed + 3. *install-deps* - the build dependencies of the package are + installed + 4. *extract-source* - the source package is extracted + 5. *build* - the proper build is executed + +## COMMANDS + +Each command accepts `--help` option that shows its basic CLI usage. + + * `bundle` [`OPTS`] [`PATH`]: Create a tarball file containing + sources of a package in the current directory, and a series of + scripts to build it using docker. The resulting *bundle* is stored + in the parent directory as a tarball. If the package is + non-native, the original tarball must be present in the parent + directory. However, if the original tarball is not present, + debocker will try to extract it using pristine-tar(1). + + The bundle is independent from debocker and can be used with + docker only (see [EXAMPLES][]). + + Arguments: + + `PATH`: optional path to the package; defaults to the current + directory + + Options: + + `-o FILE`, `--output FILE`: store bundle in `FILE`; the file is a + traditional tarball with a Debian source package, a Dockerfile, + and some utils + + `-f FLAGS`, `--flags FLAGS`: define builds flags that bundle will + use; these are passed to dpkg-buildpackage + + `--from STEP`: invalidate `STEP` causing docker to restart from + this step, even if previous cache exists; possible values are: + *build*, *extract-source*, *install-deps*, *install-utils*, + *upgrade* + + `--image IMAGE`: define the base docker image to use; defaults to + *debian:unstable* + + * `build-bundle` [`OPTS`] `BUNDLE`: Build a tarball file created + with `bundle` by running the process in docker and extracting the + final files to the current directory. This multi-step process + takes advantage of docker's cache whenever possible, saving space + and making successive builds very fast. + + Arguments: + + `BUNDLE`: the location of a bundle to build + + Options: + + `-o DIRECTORY`, `--output DIRECTORY`: store the built files in + `DIRECTORY` + + `-s`, `--sign`: sign the results of the build; this requires + installed **debsign** (see devscripts(1)) + + `--no-cache`: do not use docker's image cache (passed directly to + *docker build*); this effectively restarts the whole build from + the start + + `--pull`: pull the newest base image if available (passed directly + to *docker build*) + + * `build` [`OPTS`] [`PATH`]: Create a temporary bundle with `bundle` + and build it with `build-bundle`. The respective options are + passed unchanged to the subcommands (e.g., `-s` can be used to + sign packages). + +## OPTIONS + + * `-v`, `--verbose`: + Make debocker's output verbose. + + * `--help`: + Show summary of CLI usage. + +Global options must be given before the name of the command. + +## FILES + +There are no config files. + +## EXAMPLES + +Assuming that you are in a directory with a Debian source package: + + $ debocker build + +will build the package in Debian unstable (the built files will be +stored in the parent directory). If the build was successful, every +subsequent run should use cache instead. You may force rebuild +at any stage by using `--from` option. To rebuild the package, +you may use: + + $ debocker build --from=build + +The `build` command is equivalent, save for the undeleted, +intermediary file, with: + + $ debocker bundle --output /tmp/bundle.tar + $ debocker build-bundle /tmp/bundle.tar --output .. + +You may pass custom flags to your build: + + $ debocker build --flags='-j4' + +To create a (pseudo)-reproducible, compressed bundle with your package +and to build it using docker: + + $ debocker bundle --output /tmp/bundle.tar + $ docker run - < /tmp/bundle.tar + +## BUGS + +Debocker does not clean after itself. If you are not careful, docker +images may consume a lot of space. + +And probably many more. + +## AUTHOR + +Initial idea and coding has been done by Tomasz Buchert +. + +Initial packaging, many ideas and a lot of support by Dariusz +Dwornikowski. + +The semi-official homepage is . + +## SEE ALSO + +pbuiler(8), docker(1), devscripts(1), pristine-tar(1) diff --git a/doc/usecases-new.org b/doc/usecases-new.org new file mode 100644 index 0000000..c8867f1 --- /dev/null +++ b/doc/usecases-new.org @@ -0,0 +1,8 @@ +* BASIC: + +** debocker (re)create - creates unstable image for future builds (updated & so on) +CREATES IMAGE: debocker:unstable +** debocker (re)init - creates an image for future builds of the given package +CREATES IMAGE: debocker:{pkg} +** debocker build - builds the current package in docker +USES IMAGE: debocker:{pkg} diff --git a/doc/usecases.txt b/doc/usecases.txt new file mode 100644 index 0000000..8c89b5a --- /dev/null +++ b/doc/usecases.txt @@ -0,0 +1,89 @@ + +Debocker is for: + + * building deb packages easily + * building deb packages *reproducibly* + +1) Build a package magically. + + - build a package for the current system + + $ debocker build pkg + + - build a package for any debian release + + $ debocker build -t testing pkg + + - build a package for debian release + + $ debocker build -t ubuntu/raring pkg + + - build a particular version of a packge + + $ debocker build pkg=1.23 + + - build a particular git-tag of a package + + $ debocker build-git pkg debian/0.12.4-1 + $ debocker build-git http://github.com/thinred/new-package master + + This one uses a fact that *some* Debian packages use git to version + control packages. When you give URL, it will check the repo out + at a given tag (master by default) and build it + + - build a package from URL + + $ debocker build http://www.somewhere.com/package.debr + +Notes: + * probably puts the package in a cache or something + * when it's done, it shows the container ID + +2) Get a package at a particular version, distribution, etc. + + $ debocker get pkg=1.23 + $ debocker get-git pkg + +Notes: + * does not build, just creates a directory. Maybe initializes + 'debocker.conf' in debian/ + +3) Build current, local package sources + + $ debocker build (with no parameters) + $ debocker build --path $path (builds a package at $path) + +Notes: + + * takes the current state of the repo and builds it in a container + * signs and does all that kind of magic + +4) Package a package (sic!) in a reproducible bundle + + $ debocker bundle pkg=1.23 + $ debocker bundle (for the current directory) + +Notes: + * builds the package (to test if it builds at all) + * takes the diff and stores in a compressed file + and information how to build it later; + now, this one is tricky - docker does not allow + this, but there is a feature coming that may kind of solve it: + https://github.com/dotcloud/docker/pull/1974 + well, let's keep it for later + * it creates a file like 'bundle.debr' that contains + *all information* to build the package *reproducibly*; + normally, it should not contain the base system (it should + be referenced to some rock solid place) + + +Additional notes: + + * I imagine it could be like that; + one day I do 'debootstrap' to get an initial image; then every + day there is an 'apt-get update' to advance the image; so on the + server there is a chain of images for every day + + the developer builds a package using an automatically fetched + yesterday's tag to build on it; then he/she can distribute it + with DEB package, post it somewhere, or something like that diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..964f957 --- /dev/null +++ b/setup.py @@ -0,0 +1,51 @@ +import re +import os +from os.path import join +from codecs import open + +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + +# Thanks requests +with open('debocker', 'r') as fd: + version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', + fd.read(), re.MULTILINE).group(1) + +def files_recursive(path, dest): + store = [] + for prefix, ds, fs in os.walk(path): + final_dst = join(dest, prefix) + store.append((final_dst, [ join(prefix, f) for f in fs ])) + return store + +data_files = files_recursive('bundle-files', '/usr/share/debocker') + +setup( + name = "debocker", + version = version, + description = "debocker is a Debian packages builder using docker", + author = "Tomasz Buchert", + author_email = "tomasz@debian.org", + url = "http://anonscm.debian.org/cgit/collab-maint/debocker.git", + requires = [ 'click (>=3.3)' ], + install_requires = [ 'click>=3.3' ], + license = "GPLv3+", + zip_safe = False, + scripts = ['debocker'], + data_files = data_files, + classifiers = [ + 'Environment :: Console', + 'Development Status :: 4 - Beta', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Topic :: System :: Archiving :: Packaging', + 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)', + 'Intended Audience :: Developers', + 'Intended Audience :: System Administrators', + 'Operating System :: POSIX :: Linux', + ], +) diff --git a/tests/Makefile b/tests/Makefile new file mode 100644 index 0000000..96dca62 --- /dev/null +++ b/tests/Makefile @@ -0,0 +1,10 @@ +TEST_FILES := $(wildcard test-*.sh) +TESTS := $(TEST_FILES:.sh=.test) + +testall: $(TESTS) + +%.test: + bash $*.sh > $@ 2>&1 + +clean: + rm -f *.test diff --git a/tests/test-inception.sh b/tests/test-inception.sh new file mode 100755 index 0000000..5ea78f1 --- /dev/null +++ b/tests/test-inception.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# tests whether debocker can build itself + +set -eux + +cd .. +debocker build diff --git a/utils/debian-build/Dockerfile b/utils/debian-build/Dockerfile new file mode 100644 index 0000000..8ed8f8d --- /dev/null +++ b/utils/debian-build/Dockerfile @@ -0,0 +1,22 @@ +FROM debian:unstable +MAINTAINER Tomasz Buchert + +# Create an image that contains basic env. +# for building packages +# see 02 in steps + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get -y update && \ + apt-get -y upgrade && \ + apt-get -y --no-install-recommends install \ + devscripts pbuilder build-essential aptitude lintian && \ + apt-get clean + +# rm -rf /var/lib/apt/lists/* + +# TODO +# remove stuff below too? +# rm -rf /usr/share/doc && +# rm -rf /usr/share/man && +# rm -rf /usr/share/locale diff --git a/utils/push-new-image b/utils/push-new-image new file mode 100755 index 0000000..75b3f0b --- /dev/null +++ b/utils/push-new-image @@ -0,0 +1,11 @@ +#!/bin/bash + +set -eu + +file=$(readlink -f "$0") +here=$(dirname "$file") +cd "$here" + +docker pull debian:unstable +docker build -t debocker/unstable ./debian-build +docker push debocker/unstable:latest -- 2.20.1