You are not logged in.

#1 2021-05-08 20:46:19

loqs
Member
Registered: 2014-03-06
Posts: 13,448

W.I.P. making freedesktop-docs reproducible

The package currently is not reproducible as it uses the 'latest' version of each spec and the html can change with updates used to generate it.

I have attempted to fix the issue by generating the html locally from specific commits.  based on the script upstream uses to generate the contents https://gitlab.freedesktop.org/xdg/xdg- … web-export

# Maintainer: Sergej Pupykin <pupykin.s+arch@gmail.com>

pkgname=freedesktop-docs
pkgver=202104508
pkgrel=1
pkgdesc="Specifications from freedesktop.org"
arch=('any')
url="https://standards.freedesktop.org"
license=('GPL')
options=('docs')
makedepends=('git' 'xmlto' 'docbook-utils' 'docbook-xsl' 'docbook-xml' 'docbook-sgml' 'python-gitpython')
source=(git+https://gitlab.freedesktop.org/xdg/xdg-specs.git#commit=9a32db070a486e40b45045a22e39b8ff8905b773
        git+https://gitlab.freedesktop.org/xdg/default-icon-theme.git#commit=f522d9d0bb08c395ccc1a140bf693a9e239ef61e
        git+https://gitlab.freedesktop.org/xdg/shared-mime-info.git#commit=09faac14d0c88dc6ee50a59c7d880e9631c39ff3
        git+https://gitlab.freedesktop.org/xdg/sound-theme-spec.git#commit=b38cb3916fbb7243a3f4e78284d5613cc9321c8c
        git+https://gitlab.freedesktop.org/xdg/startup-notification.git#commit=07237ff25d6171e1b548118442ddba4259a53ba5
        git+https://gitlab.freedesktop.org/dbus/dbus.git#commit=ef55a3db0d8f17848f8a579092fb05900cc076f5
        git+https://gitlab.freedesktop.org/mpris/mpris-spec.git#commit=51e5848f9f763864568db233bffe98e3cb72bf13
        update.py)
sha256sums=('SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            'SKIP'
            '9fa781c8618de4c8cb7e7e104426830519947fd8b2f47625847d6dbe027fb245')

build() {
  cp xdg-specs/web-export/specs.idx .
  ./update.py
  find build -name '*.proc' -delete #find a way to stop these being produced?
}

package() {
  cd "$srcdir"/build
  install -d "$pkgdir"/usr/share/doc/freedesktop.org/
  cp -r --no-preserve=ownership  * "$pkgdir"/usr/share/doc/freedesktop.org/
}

update.py

#!/usr/bin/env python3

import os
import sys
import subprocess

import git
import tempfile

# Directory to put everything in, relative to the git root directory
OUTPUTDIR = "build"

# Specifications with their own build system
# FIXME: would be good to be able to use the same script to generate them
SELF_BUILT = [ "idle-inhibit-spec", "secret-service", "mpris-spec" ]

def safe_mkdir(path):
    if not dir:
        return
    os.makedirs(path,exist_ok=True)

class VcsObject:
    def __init__(self, vcstype, repo, revision):
        self.vcstype = vcstype
        if self.vcstype != 'git':
            raise Exception('Unknown VCS: %s' % self.vcs)
        self.repo = repo.split('/')[1]
        self.repo = git.Repo(self.repo)
        self.revision = revision

    def writeFile(self, outdir, path):
        basename = os.path.basename(path)
        dirname = os.path.dirname(path)
        commit = self.repo.commit(rev=self.revision)
        tree=commit.tree
        blob=tree[path]
        self.writeBlob(outdir, blob)

    def writeDir(self, outdir, path):
        commit = self.repo.commit(rev=self.revision)
        tree=commit.tree
        if path != "":
            tree=tree[path]
        self.writeTree(outdir, tree)

    def writeTree(self, outdir, tree):
        for blob in tree.blobs:
            self.writeBlob(outdir, blob)
        for subtree in tree.trees:
            subdir=os.path.join(outdir, subtree.name)
            safe_mkdir(subdir)
            self.writeTree(subdir, subtree)

    def writeBlob(self, outdir, blob):
        filename = os.path.join(outdir, blob.name)
        with open(filename,"wb") as fp:
            fp.write(blob.data_stream.read())

class SpecObject():
    def __init__(self, vcs, path, spec, out_dir, version):
        self.vcs = vcs
        self.spec = spec
        self.version = version
        self.path = path
        self.out_dir= os.path.abspath(os.path.join(out_dir, spec, version))
        safe_mkdir(self.out_dir)
        self.working_dir=tempfile.TemporaryDirectory()
        self.ext = os.path.splitext(self.path)[1]

        if self.ext not in ['.xml', '.sgml', '.txt', '.dtd', '.html']:
            raise Exception('Format \'%s\' not supported for %s' % (self.ext, self.vcs.get_url()))

    def _run(self,args):
        result = subprocess.run(args, cwd=self.working_dir.name)
        result.check_returncode()

    def downloadDir(self,path):
        out_dir= self.working_dir.name
        self.vcs.writeDir(out_dir, path)

    def generate(self):
        one_chunk_command = None
        multiple_chunks_command = None

        if self.spec not in SELF_BUILT:
            if self.ext in ('.dtd', '.hml', '.txt'):
                self.vcs.writeFile(self.out_dir, self.path)
                return
            else:
                self.vcs.writeFile(self.working_dir.name, self.path)
                if self.ext == '.xml':
                    one_chunk_command = ['xmlto', '-o', 'html-nochunks', os.path.basename(self.path)]
                    multiple_chunks_command = ['xmlto', '-o', 'html', os.path.basename(self.path)]
                elif self.ext == '.sgml':
                    one_chunk_command = ['docbook2html', '-o', '--nochunks', os.path.basename(self.path)]
                    multiple_chunks_command = ['docbook2html', '-o', os.path.basename(self.path)]
        else:
            if self.spec == 'mpris-spec':
                return #Not supported
                self.downloadDir("")
            else:
                self.downloadDir(os.path.dirname(self.path))
                if self.spec == 'idle-inhibit-spec':
                    args = ['xsltproc', '-o', 'reference.xml', 'tools/spec-to-docbook.xsl', 'org.freedesktop.ScreenSaver.xml']
                    self._run(args)
                    one_chunk_command = ['xmlto', '-o', '--skip-validation', 'xhtml-nochunks', 'specification.xml']
                    multiple_chunks_command = ['xmlto', '-o', '--skip-validation', '-x', 'docbook-params.xsl', 'xhtml', 'specification.xml']
                elif self.spec == 'secret-service':
                    args = ['xsltproc', '-o', 'reference.xml', 'tools/spec-to-docbook.xsl', 'org.freedesktop.Secrets.xml']
                    self._run(args)
                    one_chunk_command = ['xmlto', '-o', '--skip-validation', 'xhtml-nochunks', 'specification.xml']
                    multiple_chunks_command = ['xmlto', '-o', '--skip-validation', '-x', 'docbook-params.xsl', 'xhtml', 'specification.xml']

        if self.spec in SELF_BUILT:
            print("Converting", self.spec,self.version, "to HTML")
        else:
            print("Converting", self.path,self.version, "to HTML")

        if one_chunk_command:
            one_chunk_command.insert(2, self.out_dir)
            self._run(one_chunk_command)

        if multiple_chunks_command:
            html_dir= os.path.join(self.out_dir, "html-multi")
            safe_mkdir(html_dir)
            multiple_chunks_command.insert(2, html_dir)
            self._run(multiple_chunks_command)

safe_mkdir(OUTPUTDIR)
with open('specs.idx') as specs:
    for line in specs:
        line = line.strip()
        if not line or line.startswith('#'):
            continue

        (data, revision, version, spec) = line.split()
        (vcstype, repo, path) = data.split(":")
        if vcstype == "git":
            vcs = VcsObject('git', repo, revision)
        else:
            raise Exception('VCS Format \'%s\' not supported for %s' % (vcstype, repo))

        spec = SpecObject(vcs, path, spec, OUTPUTDIR, version)
        spec.generate()

specs.idx

git:xdg/xdg-specs:autostart/autostart-spec.xml                 HEAD            0.5	autostart-spec

# git:xdg/xdg-specs:basedir/basedir-spec.xml		HEAD	0.9	basedir-spec
git:xdg/xdg-specs:basedir/basedir-spec.xml		373ecf4423f98384ec299268cc10c53afec418cf	0.8	basedir-spec
git:xdg/xdg-specs:basedir/basedir-spec.xml		e9be1b7628eb1db3e85be02a3a9f424448be1ec6	0.7	basedir-spec
git:xdg/xdg-specs:basedir/basedir-spec.xml		2c096d0c74244bd573027b217be3a69335a923f4	0.6	basedir-spec
# Invalid docbook:
#git:xdg/xdg-specs:basedir/basedir-spec.xml		c8475bef1d1f84afcc89c710b6b5b214f11a670f	0.5	basedir-spec

git:xdg/xdg-specs:clipboard/clipboards.txt		e7b4103d0019c6ccfe2651ba63db8cf09e19ee80	0.1	clipboards-spec
git:xdg/xdg-specs:clipboard/clipboard-extensions.txt		791f359c0a03dd36e716a9a040a2e4ea620b4059	0.1	clipboard-extensions-spec

git:dbus/dbus:doc/introspect.dtd				7652304bff969afb3969603149bb385efe861fe8		1.0	dbus

git:xdg/xdg-specs:desktop-entry/desktop-entry-spec.xml          HEAD          1.5     desktop-entry-spec
git:xdg/xdg-specs:desktop-entry/desktop-entry-spec.xml          31ba7763d443c34deb68a432d5d634207e0667a2          1.4     desktop-entry-spec
git:xdg/xdg-specs:desktop-entry/desktop-entry-spec.xml          839580ee24a06b578c2b635fd7400001f5bfc6f4          1.3     desktop-entry-spec
git:xdg/xdg-specs:desktop-entry/desktop-entry-spec.xml          7bec5bec45c13653eb614ce084ace93d9b5c8244          1.2     desktop-entry-spec
git:xdg/xdg-specs:desktop-entry/desktop-entry-spec.xml          e9be1b7628eb1db3e85be02a3a9f424448be1ec6          1.1     desktop-entry-spec
git:xdg/xdg-specs:desktop-entry/desktop-entry-spec.xml          e1775a202af19adf9c73e6954293e2e95f8f13a0          1.0     desktop-entry-spec
git:xdg/xdg-specs:desktop-entry/desktop-entry-spec.xml          7d249792235d0f3b69f833f89bef545a367f0b6b          0.9.8     desktop-entry-spec
git:xdg/xdg-specs:desktop-entry/desktop-entry-spec.xml          3e675b52041139085cc280dcdf2562e38cba8a31          0.9.7     desktop-entry-spec
git:xdg/xdg-specs:desktop-entry/desktop-entry-spec.xml          af1b346daa90b9cb88ec9e3b6d3ebd05942a73d7          0.9.6     desktop-entry-spec
git:xdg/xdg-specs:desktop-entry/desktop-entry-spec.xml          6855384d021d88557f80674798c584c15b547f36          0.9.5     desktop-entry-spec
# Invalid docbook:
#git:xdg/xdg-specs:desktop-entry/desktop-entry-spec.xml          b818b36ff0ee3c7f35df7b38db15a18b434f6375          0.9.4     desktop-entry-spec
git:xdg/xdg-specs:desktop-entry/desktop-entry-spec.sgml          e7d2b4770815f8d8dcca934d3520d3fe55f86b09          0.9.3     desktop-entry-spec
git:xdg/xdg-specs:desktop-entry/desktop-entry-spec.sgml          9c493d5fca5fcceca4e1bc0b2b2ba8a13c5eb12e          0.9.2     desktop-entry-spec

git:xdg/default-icon-theme:spec/icon-naming-spec.xml    HEAD          0.8.90  icon-naming-spec
git:xdg/default-icon-theme:spec/icon-naming-spec.xml    77bc8ba641ca24bf2e55292f7d31926d60295e5c          0.8  icon-naming-spec
git:xdg/default-icon-theme:spec/icon-naming-spec.xml    aba09c448ecc055f93a959a54497ef02be4ceab2          0.7  icon-naming-spec
git:xdg/default-icon-theme:spec/icon-naming-spec.xml    678139104048733b680c331d28ee09f6f16f386c          0.6  icon-naming-spec
git:xdg/default-icon-theme:spec/icon-naming-spec.xml    1dc55c9db340a7d0a9ac84e1e42735df311cdad5          0.4  icon-naming-spec
# Invalid docbook:
#git:xdg/default-icon-theme:spec/icon-naming-spec.xml    db9bb955c0cf20322b149676b908ecd69929d2f1          0.3  icon-naming-spec

# FIXME does not support rev != HEAD
git:xdg/xdg-specs:idle-inhibit/specification.xml       HEAD                                            0.1   idle-inhibit-spec
git:xdg/xdg-specs:secret-service/specification.xml     HEAD                                            0.2   secret-service
git:mpris/mpris-spec:spec/all.xml                      HEAD                                            2.2   mpris-spec

git:xdg/default-icon-theme:spec/icon-theme-spec.xml    fca8fecff452ed188584f22e99e9b96458f697f9          0.13  icon-theme-spec
git:xdg/default-icon-theme:spec/icon-theme-spec.xml    7d2735da96abf7e1d2ce9a56e38240498e2f5e67          0.12  icon-theme-spec
git:xdg/default-icon-theme:spec/icon-theme-spec.xml    f09150e444382a18d147ef45afa9310e2e27c713          0.11  icon-theme-spec
git:xdg/default-icon-theme:spec/icon-theme-spec.xml    37f3118e81954b2171a333614901a0b74c295edc          0.10  icon-theme-spec
git:xdg/default-icon-theme:spec/icon-theme-spec.xml    400de5cb3e4f8046763ce816fb93a9ec287e6e39          0.9  icon-theme-spec
git:xdg/default-icon-theme:spec/icon-theme-spec.xml    61ce5bd0f09d5026b32f015f7f8d209524b85cf1          0.8  icon-theme-spec
git:xdg/default-icon-theme:spec/icon-theme-spec.xml    bd6077c5fc235d830d325a977770150bd2d71136          0.7  icon-theme-spec
git:xdg/default-icon-theme:spec/icon-theme-spec.xml    b1a891e95eb4f28da4e43fb6cb7a662bc0431529          0.6  icon-theme-spec

git:xdg/xdg-specs:mime-apps/mime-apps-spec.xml		1354abdb635b560f3c36373c8074ee7e5f63abab		1.0	mime-apps-spec
git:xdg/xdg-specs:mime-apps/mime-apps-spec.xml		HEAD		1.0.1	mime-apps-spec

git:xdg/xdg-specs:menu/menu-spec.xml			HEAD		1.1	menu-spec
git:xdg/xdg-specs:menu/menu-spec.xml			647153fb87e5f642a7b7a3fdcec90961f36e89ab		1.0	menu-spec
git:xdg/xdg-specs:menu/menu-spec.xml			a61b06aa1aee4743f50024b0695795fe9568d929		0.92	menu-spec
git:xdg/xdg-specs:menu/menu-spec.xml			2403355e486ac6bc972fd150d38e3c44d086960a		0.91	menu-spec
git:xdg/xdg-specs:menu/menu-spec.xml			e1b6bf680dc4d021e7f2122841f812975a31985c		0.9	menu-spec
# Invalid docbook:
#git:xdg/xdg-specs:menu/menu-spec.xml			7d616706bba168427214848c6fbb77da5a30e04d		0.8	menu-spec
#git:xdg/xdg-specs:menu/menu-spec.xml			28a4c60cb40502c01417bfdbd8a2e093eb4af2d9		0.7	menu-spec
#git:xdg/xdg-specs:menu/menu-spec.xml			9f969d0e6625b2da24fb971ebd3cf7f913a1c5f1		0.6	menu-spec
#git:xdg/xdg-specs:menu/menu-spec.xml			c15b9df59efac72ea80fb715bf2a9d7f7afd1bba		0.5	menu-spec
#git:xdg/xdg-specs:menu/menu-spec.xml			c8475bef1d1f84afcc89c710b6b5b214f11a670f		0.4	menu-spec

# XXX: Note that the version should stay "1.0" so it is exported as
# http://www.freedesktop.org/standards/menu-spec/menu-1.0.dtd
# as per the specification
git:xdg/xdg-specs:menu/menu.dtd                                2403355e486ac6bc972fd150d38e3c44d086960a                1.0	menu-spec

git:xdg/xdg-specs:notification/notification-spec.xml	HEAD	1.2	notification-spec

git:xdg/xdg-specs:recent-files/recent-file-spec.xml		2f2e0cc7ded9b2c8afe29281c621d8ee6c5def9a	0.2	recent-file-spec

git:xdg/shared-mime-info:shared-mime-info-spec.xml	90eb7974ec4e8ea9ae7daa96b1d2f9a2b4fe9a20		0.21	shared-mime-info-spec
git:xdg/shared-mime-info:shared-mime-info-spec.xml	2f909cff817a3f08d12321769173cac2de0da1b4		0.20	shared-mime-info-spec
git:xdg/shared-mime-info:shared-mime-info-spec.xml	30349a8810143c1f808a74c96bceba87d48e8364		0.19	shared-mime-info-spec
git:xdg/shared-mime-info:shared-mime-info-spec.xml	1bac8224c7439452a85693dc1a25f64d3a467e7d		0.18	shared-mime-info-spec
git:xdg/shared-mime-info:shared-mime-info-spec.xml	83c091bbb4d32231b308864d227ada6106bc59a5		0.17	shared-mime-info-spec
git:xdg/shared-mime-info:shared-mime-info-spec.xml	6c474418318c6c192abfe0f17f85209c65bfa1cf		0.16	shared-mime-info-spec
git:xdg/shared-mime-info:shared-mime-info-spec.xml	7fab0df72e558a1d651e2557d52ac37b8d5020e9		0.15	shared-mime-info-spec
git:xdg/shared-mime-info:shared-mime-info-spec.xml	0239582644c9edffa7edd4562ccb967d14d63e73		0.14	shared-mime-info-spec
git:xdg/shared-mime-info:shared-mime-info-spec.xml	f364fdbba5bb7fe8b69e0294aa353845515b191e		0.13	shared-mime-info-spec
git:xdg/shared-mime-info:shared-mime-info-spec.xml	ad3203089d3d029b5c18ebd4baa6f7991dd9bb22		0.12	shared-mime-info-spec
git:xdg/shared-mime-info:shared-mime-info-spec.xml	92cca5e4cec157f252fd77b5ce23853974603906		0.11	shared-mime-info-spec
git:xdg/shared-mime-info:shared-mime-info-spec.xml	dc2724f1d1b010a80b269ef21a1e613f55170777		0.10	shared-mime-info-spec
git:xdg/shared-mime-info:shared-mime-info-spec.xml	e1359233cf0544509854178e8591ed39b5ccbcdf		0.9	shared-mime-info-spec
git:xdg/shared-mime-info:shared-mime-info-spec.xml	7b25e407e6dc6cda94c2fb1f83b4ac4956a22fbf		0.8	shared-mime-info-spec
git:xdg/shared-mime-info:shared-mime-info-spec.xml	d08ddd99abf32cbf770fecc43f950f4a8c7a4dc0		0.7	shared-mime-info-spec
git:xdg/shared-mime-info:shared-mime-info-spec.xml	1dd7f6ce4610378b970674cfef61e1bda2389ba5		0.6	shared-mime-info-spec
git:xdg/shared-mime-info:shared-mime-info-spec.xml	2e3dfe6117ec97da5493669a05459fdcac914e1c		0.5	shared-mime-info-spec
git:xdg/shared-mime-info:shared-mime-info-spec.xml	bfd730e142344783603ac4cdf0467ccf2e2ff1fd		0.4	shared-mime-info-spec
git:xdg/shared-mime-info:shared-mime-info-spec.xml	f6fa8bf89055da53e9323aca59d5337729178ade		0.3	shared-mime-info-spec
git:xdg/shared-mime-info:shared-mime-info-spec.xml	e235408efaeec39b5be22d4f1d4218da64247d74		0.2	shared-mime-info-spec

git:xdg/sound-theme-spec:spec/sound-naming-spec.xml	HEAD							0.2	sound-naming-spec
git:xdg/sound-theme-spec:spec/sound-theme-spec.xml	HEAD							0.2	sound-theme-spec

git:xdg/startup-notification:doc/startup-notification.txt   HEAD                0.2     startup-notification-spec
git:xdg/startup-notification:doc/startup-notification.txt   e1d689b36635ed632cbb91cf3104f84bd0b59bfd                0.1     startup-notification-spec

git:xdg/xdg-specs:systemtray/systemtray-spec.xml			cbe62e957475dba73d9d830dd755c76f8d3abd23		0.3	systemtray-spec
git:xdg/xdg-specs:systemtray/systemtray-spec.xml			87fb46a8631a8b509bb1b8026b88c7059f869ffd		0.2	systemtray-spec
git:xdg/xdg-specs:systemtray/systemtray-spec.xml			c8475bef1d1f84afcc89c710b6b5b214f11a670f		0.1	systemtray-spec

git:xdg/xdg-specs:thumbnail/thumbnail-spec.sgml			HEAD	0.9.0	thumbnail-spec
git:xdg/xdg-specs:thumbnail/thumbnail-spec.sgml			e9be1b7628eb1db3e85be02a3a9f424448be1ec6	0.8.0	thumbnail-spec
git:xdg/xdg-specs:thumbnail/thumbnail-spec.sgml			27347f5fe2c06fe95f19a041941036a9f7413a08	0.7.0	thumbnail-spec

git:xdg/xdg-specs:trash/trashspec.html			HEAD		1.0	trash-spec

git:xdg/xdg-specs:wm-spec/wm-spec.xml			HEAD		1.5	wm-spec
git:xdg/xdg-specs:wm-spec/wm-spec.xml			a95af55a102b0814e81095bda46a691068009c3c		1.4	wm-spec
git:xdg/xdg-specs:wm-spec/wm-spec.xml			2eafacb23421d25147533a66a9eb8f0f4650a5e4		1.3	wm-spec
git:xdg/xdg-specs:wm-spec/wm-spec.sgml			d10d819f39c4c7e878288192f14248536d9d4b96		1.2	wm-spec
git:xdg/xdg-specs:wm-spec/wm-spec.sgml			a6e9a0a88112e5f6ac7099530e5c732c7ec6fb7f		1.1	wm-spec
# Invalid docbook:
#git:xdg/xdg-specs:wm-spec/wm-spec.sgml			b58352845cebc1d3b5754215a03ed047d1b28b11		1.0	wm-spec

git:xdg/xdg-specs:xembed/spec/xembed-spec.xml		7bcfe52e21363072766a7ab428a9ab3e8552f6ff	0.5	xembed-spec

git:xdg/xdg-specs:xsettings/xsettings.xml		481e6fef65998dd376084532af878268cd45ae82	0.5	xsettings-spec

Outstanding issues:
How to set the pkgver.
How to sync specs.idx.
Is there anyway to produce the mpris spec using python3 instead of python2?  Currently it is not built.
Edit:
Fixed missing dbus repo.

Last edited by loqs (2021-05-09 17:05:08)

Offline

Board footer

Powered by FluxBB