Arch Linux

solstice

Member

Registered: 2006-10-27

Posts: 239

Website

quick script to check integrity of files

hi . i stumbled upon this feature request http://bugs.archlinux.org/task/11091?project=3
and then coded a python script that check that installed files of packages are the same as in the downloaded package in the cache.

beware it's a quick script. and not a security software like tripwire (non free), aide, or samhain or whatever...

it can take parameter from a pipe like

tail -n 10 /var/log/pacman.log |grep upgraded|cut -f 4 -d ' '|python verifypkg.py

or

python verifypkg.py /var/cache/pacman/pkg/xorg-utils-7.5-1-i686.pkg.tar.gz

here is an updated version of that script

#!/usr/bin/env python

# verify.py                                          Version 0.1      2009-10-18
#
# under the WTFPL. see http://sam.zoy.org/wtfpl/
#
#             DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE 
#                     Version 2, December 2004 
# 
#  Copyright (C) 2006 solsTiCe d'Hiver               <solstice.dhiver@gmail.com>
#  Everyone is permitted to copy and distribute verbatim or modified 
#  copies of this license document, and changing it is allowed as long 
#  as the name is changed. 
# 
#             DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE 
#    TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 
# 
#   0. You just DO WHAT THE FUCK YOU WANT TO.

'''a python script that compare the files in a package tarball and the files
installed on the system:
    It reports changed files, missing files'''

import tarfile
import hashlib
from sys import argv, exit, stderr, stdin
from os import listdir
from os.path import exists, basename
from glob import glob

DB = '/var/lib/pacman'
CACHE = '/var/cache/pacman/pkg'

# TODO: use logging module ?
def error(s):
    stderr.write('Error: %s\n' % s)

def warning(s):
    stderr.write('Warning: %s\n' % s)

def getpkgname(pkgfile):
    '''return package name from its pkgfilename'''
    pkg = basename(pkgfile.rstrip('.pkg.tar.gz'))
    pkg = pkg[:pkg.rfind('-')]
    return pkg

def ispkginstalled(pkgfile):
    '''return wether or not a package is installed'''
    return exists('/'.join([DB, 'local', getpkgname(pkgfile)]))

def checkpkg(pkgfile):
    '''check if the installed files of a package have changed compared to the
    files in the package tarball'''
    missing = 0
    changed = 0
    md5s = md5pkg(pkgfile)
    for filename,md5 in md5s.items():
        ff = '/' + filename
        if exists(ff):
            try:
                with open(ff) as g:
                    if hashlib.md5(g.read()).hexdigest() != md5:
                        warning('%s has changed' % ff)
                        changed += 1
            except IOError as e:
                error('%s: %s ' % (e.strerror, filename))
                continue
        else:
            warning('%s has not been found' % ff)
            missing += 1
    return (len(md5s), changed, missing)

def listpkg():
    '''return a list of all isntalled pkg'''
    return sorted(listdir('/'.join([DB, 'local'])))

def findpkg(s, dbpkg):
    '''look for an installed pkg matching s and return the path of tarball in
    the cache'''
    candidates = [p for p in dbpkg if p.startswith(s)]
    found = False
    for i in candidates:
        pkgname = '-'.join(i.split('-')[:-2])
        if pkgname == s:
            found = True
            break
    if found:
        res = glob('/'.join([CACHE, i+'*']))
        if len(res) == 1:
            return res[0]
    return None

def md5pkg(pkgfile):
    '''return a dictionary of filename and md5 of the package'''
    tf = tarfile.open(pkgfile, 'r')
    md5s = {}
    for ti in tf:
        if ti.isfile() and ti.name not in ('.PKGINFO', '.INSTALL', '.CHANGELOG'):
            f = tf.extractfile(ti)
            md5s[ti.name] = hashlib.md5(f.read()).hexdigest()
            f.close()
    tf.close()
    return md5s

if __name__ == '__main__':
    dbpkg = listpkg()
    try:
        if len(argv) == 1:
            # try to use stdin if no argument
            pkglist = stdin.readlines()
            pkglist = [i.strip() for i in pkglist]
        else:
            pkglist = argv[1:]
        for pkg in pkglist:
            if exists(pkg):
                if not ispkginstalled(pkg):
                    error('%s is not installed' % getpkgname(pkg))
                    continue
            else:
                # if we do not find the archive pkg, then look for a package with
                # that name in the db of pacman
                p = findpkg(pkg, dbpkg)
                if p != None:
                    # strange but it works
                    pkg = p
                else:
                    error('%s not found' % pkg)
                    continue
            # finaly check the tarball pkg
            (n,c,m) = checkpkg(pkg)
            print '%s: %d files, %d changed, %d missing' % (getpkgname(pkg), n, c, m)
    except KeyboardInterrupt:
        pass

Last edited by solstice (2009-11-02 15:00:51)

solstice

Member

Registered: 2006-10-27

Posts: 239

Website

Re: quick script to check integrity of files

A new version that simplify it's use

python2 verifypkg.py

It will look for all packages in pacman cache and check the integrety of each of its files

or you could check only a given package, for example:

 $ python2 verifypkg.py coreutils-8.13-2
coreutils-8.13-2: 240 files, 0 changed, 0 missing

you need to use the complete name of the package with version and release

http://paste.pocoo.org/show/485669/

#!/usr/bin/env python2

# verify.py                                          Version 0.2      2009-10-18
#
# under the WTFPL. see http://sam.zoy.org/wtfpl/
#
#             DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE 
#                     Version 2, December 2004 
# 
#  Copyright (C) 2006 solsTiCe d'Hiver               <solstice.dhiver@gmail.com>
#  Everyone is permitted to copy and distribute verbatim or modified 
#  copies of this license document, and changing it is allowed as long 
#  as the name is changed. 
# 
#             DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE 
#    TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 
# 
#   0. You just DO WHAT THE FUCK YOU WANT TO.

'''a python script that compare the files in a package tarball and the files
installed on the system:
    It reports changed files, missing files'''

import sys
import os
import tarfile
from glob import glob
from StringIO import StringIO
import hashlib
try:
    import lzma
except ImportError:
    print >>sys.stderr, 'You need the python bindings for lzma: python-pyliblzma or pylzma'

DB = '/var/lib/pacman'
CACHE = '/var/cache/pacman/pkg'

# TODO: use logging module ?
def error(s):
    sys.stderr.write('Error: %s\n' % s)

def warning(s):
    sys.stderr.write('Warning: %s\n' % s)

def getpkgname(pkgfile):
    '''return package name from its pkgfilename'''
    pkg = '.'.join(os.path.basename(pkgfile).split('.')[:-3]) # remove trailing .pkg.tar.gz
    pkg = '-'.join(pkg.split('-')[:-1])  # remove arch field
    return pkg

def ispkginstalled(pkgfile):
    '''return wether or not a package is installed'''
    return os.path.exists('/'.join([DB, 'local', getpkgname(pkgfile)]))

def checkpkg(pkgfile):
    '''check if the installed files of a package have changed compared to the
    files in the package tarball'''
    missing = 0
    changed = 0
    md5s = pkgmd5s(pkgfile)
    for filename,md5 in md5s.items():
        ff = '/' + filename
        if os.path.exists(ff):
            try:
                with open(ff) as g:
                    if hashlib.md5(g.read()).hexdigest() != md5:
                        warning('%s has changed' % ff)
                        changed += 1
            except IOError as e:
                error('%s: %s ' % (e.strerror, filename))
                continue
        else:
            warning('%s has not been found' % ff)
            missing += 1
    return (len(md5s), changed, missing)

def listpkg():
    '''return a list of all installed pkg'''
    return sorted(os.listdir('/'.join([DB, 'local'])))

def findpkgtarball(s, dbpkg):
    '''look for an installed pkg matching s and return the path of tarball in
    the cache'''
    candidates = [p for p in dbpkg if p.startswith(s)]
    found = False
    if s in candidates:
        found = True
        name = s
    else:
        for i in candidates:
            pkgname = '-'.join(i.split('-')[:-2])
            if pkgname == s:
                found = True
                name = i
                break
    if found:
        res = glob('/'.join([CACHE, name+'*']))
        if len(res) == 1:
            return res[0]
    return None

def pkgmd5s(pkgfile):
    '''return a dictionary of filename and md5 of the package'''
    if pkgfile.endswith('xz'):
        with open(pkgfile, 'r') as f:
            tf = tarfile.TarFile(fileobj=StringIO(lzma.decompress(f.read())))
    else:
        tf = tarfile.open(pkgfile, 'r')
    md5s = {}
    for ti in tf:
        if ti.isfile() and ti.name not in ('.PKGINFO', '.INSTALL', '.CHANGELOG'):
            f = tf.extractfile(ti)
            md5s[ti.name] = hashlib.md5(f.read()).hexdigest()
            f.close()
    tf.close()
    return md5s

if __name__ == '__main__':
    try:
        pkglist = sys.argv[1:]

        dbpkg = listpkg()
        if pkglist == []:
            pkglist = dbpkg

        for pkg in pkglist:
            if os.path.exists(pkg):
                if not ispkginstalled(pkg):
                    error('%s is not installed' % getpkgname(pkg))
                    continue
            else:
                # if we do not find the archive pkg, then look for a package with
                # that name in the db of pacman
                p = findpkgtarball(pkg, dbpkg)
                if p != None:
                    pkg = p
                else:
                    error('%s not found in the cache' % pkg)
                    continue
            # finaly check the tarball pkg
            (n,c,m) = checkpkg(pkg)
            print '%s: %d files, %d changed, %d missing' % (getpkgname(pkg), n, c, m)
    except KeyboardInterrupt:
        pass

Last edited by solstice (2011-10-01 17:31:19)