From d5835d4300a1db5b58ceff7e945c4a017fce13d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Thu, 27 Feb 2014 22:08:30 +0200 Subject: [PATCH] Further Python 3 porting Thanks to Alberto Planas Dominguez for sanity checks and suggestions. --- BinariesCheck.py | 4 ++-- DocFilesCheck.py | 3 ++- FilesCheck.py | 54 +++++++++++++++++++++++++--------------------- INSTALL | 4 ++-- InitScriptCheck.py | 8 +++---- LSBCheck.py | 9 +++----- MenuCheck.py | 8 +++---- Pkg.py | 39 ++++++++++++++++++++------------- TagsCheck.py | 13 ++++------- rpmdiff | 8 +++---- 10 files changed, 78 insertions(+), 72 deletions(-) diff --git a/BinariesCheck.py b/BinariesCheck.py index 5a3aab47..5380640d 100644 --- a/BinariesCheck.py +++ b/BinariesCheck.py @@ -178,7 +178,7 @@ class BinaryInfo: try: fobj = open(path, 'rb') fobj.seek(-12, 2) # 2 == os.SEEK_END, for python 2.4 compat (#172) - self.tail = fobj.read().decode() + self.tail = Pkg.b2s(fobj.read()) except Exception: e = sys.exc_info()[1] printWarning(pkg, 'binaryinfo-tail-failed %s: %s' % (file, e)) @@ -274,7 +274,7 @@ class BinariesCheck(AbstractCheck.AbstractCheck): multi_pkg = False srpm = pkg[rpm.RPMTAG_SOURCERPM] if srpm: - res = srcname_regex.search(srpm.decode()) + res = srcname_regex.search(Pkg.b2s(srpm)) if res: multi_pkg = (pkg.name != res.group(1)) diff --git a/DocFilesCheck.py b/DocFilesCheck.py index 3cf8d966..2887f293 100644 --- a/DocFilesCheck.py +++ b/DocFilesCheck.py @@ -17,6 +17,7 @@ import rpm from Filter import addDetails, printWarning +from Pkg import b2s import AbstractCheck @@ -44,7 +45,7 @@ class DocFilesCheck(AbstractCheck.AbstractCheck): # register things which are provided by the package for i in pkg.header[rpm.RPMTAG_PROVIDES]: - core_reqs[i.decode()] = [] + core_reqs[b2s(i)] = [] for i in files: core_reqs[i] = [] diff --git a/FilesCheck.py b/FilesCheck.py index 320c1d0d..3adaab3d 100644 --- a/FilesCheck.py +++ b/FilesCheck.py @@ -12,13 +12,12 @@ from datetime import datetime import os import re import stat -import string import sys import rpm from Filter import addDetails, printError, printWarning -from Pkg import catcmd, getstatusoutput, is_utf8, is_utf8_str +from Pkg import b2s, catcmd, getstatusoutput, is_utf8, is_utf8_str import AbstractCheck import Config @@ -211,7 +210,7 @@ lib_path_regex = re.compile('^(/usr(/X11R6)?)?/lib(64)?') lib_package_regex = re.compile('^(lib|.+-libs)') hidden_file_regex = re.compile('/\.[^/]*$') manifest_perl_regex = re.compile('^/usr/share/doc/perl-.*/MANIFEST(\.SKIP)?$') -shebang_regex = re.compile('^#!\s*(\S+)') +shebang_regex = re.compile(b'^#!\s*(\S+)') interpreter_regex = re.compile('^/(usr/)?(s?bin|games|libexec(/.+)?|(lib(64)?|share)/.+)/[^/]+$') script_regex = re.compile('^/((usr/)?s?bin|etc/(rc\.d/init\.d|X11/xinit\.d|cron\.(hourly|daily|monthly|weekly)))/') sourced_script_regex = re.compile('^/etc/(bash_completion\.d|profile\.d)/') @@ -246,15 +245,21 @@ man_nowarn_regex = re.compile( r'(can\'t break|cannot adjust) line') man_warn_category = Config.getOption('ManWarningCategory', 'mac') -fsf_license_regex = re.compile('(GNU((\s+(Library|Lesser|Affero))?(\s+General)?\s+Public|\s+Free\s+Documentation)\s+Licen[cs]e|(GP|FD)L)', re.IGNORECASE) -fsf_wrong_address_regex = re.compile('(675\s+Mass\s+Ave|59\s+Temple\s+Place|Franklin\s+Steet|02139|02111-1307)', re.IGNORECASE) +fsf_license_regex = re.compile(b'(GNU((\s+(Library|Lesser|Affero))?(\s+General)?\s+Public|\s+Free\s+Documentation)\s+Licen[cs]e|(GP|FD)L)', re.IGNORECASE) +fsf_wrong_address_regex = re.compile(b'(675\s+Mass\s+Ave|59\s+Temple\s+Place|Franklin\s+Steet|02139|02111-1307)', re.IGNORECASE) scalable_icon_regex = re.compile(r'^/usr(?:/local)?/share/icons/.*/scalable/') -# loosely inspired from Python Cookbook -# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/173220 -text_characters = "".join(map(chr, range(32, 127))) + "\n\r\t\b" -_null_trans = string.maketrans("", "") +# "is binary" stuff borrowed from https://pypi.python.org/pypi/binaryornot +# TODO: switch to it sometime later instead of embedding our own copy + +printable_extended_ascii = b'\n\r\t\f\b' +if bytes is str: + # Python 2 means we need to invoke chr() explicitly + printable_extended_ascii += b''.join(map(chr, range(32, 256))) +else: + # Python 3 means bytes accepts integer input directly + printable_extended_ascii += bytes(range(32, 256)) def peek(filename, pkg, length=1024): """Peek into a file, return a chunk from its beginning and a flag if it @@ -272,7 +277,7 @@ def peek(filename, pkg, length=1024): fobj.close() return (chunk, False) - if "\0" in chunk: + if b'\0' in chunk: return (chunk, False) if not chunk: # Empty files are considered text @@ -281,18 +286,17 @@ def peek(filename, pkg, length=1024): fl = filename.lower() # PDF's are binary but often detected as text by the algorithm below - if fl.endswith('.pdf') and chunk.startswith('%PDF-'): + if fl.endswith('.pdf') and chunk.startswith(b'%PDF-'): return (chunk, False) # Ditto RDoc RI files if fl.endswith('.ri') and '/ri/' in fl: return (chunk, False) - # Get the non-text characters (maps a character to itself then - # use the 'remove' option to get rid of the text characters.) - t = chunk.translate(_null_trans, text_characters) + # Binary if control chars are > 30% of the string + control_chars = chunk.translate(None, printable_extended_ascii) + nontext_ratio = float(len(control_chars)) / float(len(chunk)) + istext = nontext_ratio <= 0.30 - # If more than 30% non-text characters, then consider it a binary file - istext = float(len(t))/len(chunk) <= 0.30 return (chunk, istext) # See Python sources for a full list of the values here. @@ -523,12 +527,12 @@ class FilesCheck(AbstractCheck.AbstractCheck): oct(perm)) # Prefetch scriptlets, strip quotes from them (#169) - postin = (pkg[rpm.RPMTAG_POSTIN] or \ - pkg.scriptprog(rpm.RPMTAG_POSTINPROG)).decode() + postin = b2s(pkg[rpm.RPMTAG_POSTIN]) or \ + pkg.scriptprog(rpm.RPMTAG_POSTINPROG) if postin: postin = quotes_regex.sub('', postin) - postun = (pkg[rpm.RPMTAG_POSTUN] or \ - pkg.scriptprog(rpm.RPMTAG_POSTUNPROG)).decode() + postun = b2s(pkg[rpm.RPMTAG_POSTUN]) or \ + pkg.scriptprog(rpm.RPMTAG_POSTUNPROG) if postun: postun = quotes_regex.sub('', postun) @@ -556,7 +560,7 @@ class FilesCheck(AbstractCheck.AbstractCheck): if chunk: res = shebang_regex.search(chunk) if res: - interpreter = res.group(1) + interpreter = b2s(res.group(1)) if doc_regex.search(f): if not interpreter: @@ -608,8 +612,8 @@ class FilesCheck(AbstractCheck.AbstractCheck): elif not install_info_regex.search(postin): printError(pkg, 'postin-without-install-info', f) - preun = pkg[rpm.RPMTAG_PREUN] or \ - pkg.scriptprog(rpm.RPMTAG_PREUNPROG) + preun = b2s(pkg[rpm.RPMTAG_PREUN]) or \ + pkg.scriptprog(rpm.RPMTAG_PREUNPROG) if not postun and not preun: printError(pkg, 'info-files-without-install-info-postun', f) @@ -815,11 +819,11 @@ class FilesCheck(AbstractCheck.AbstractCheck): if not mode_is_exec and not is_doc: printError(pkg, 'non-executable-script', f, oct(perm), interpreter) - if '\r' in chunk: + if b'\r' in chunk: printError( pkg, 'wrong-script-end-of-line-encoding', f) elif is_doc and not skipdocs_regex.search(f): - if '\r' in chunk: + if b'\r' in chunk: printWarning( pkg, 'wrong-file-end-of-line-encoding', f) # We check only doc text files for UTF-8-ness; diff --git a/INSTALL b/INSTALL index f157d5a2..ea722913 100644 --- a/INSTALL +++ b/INSTALL @@ -1,6 +1,6 @@ You need the following utilities to run rpmlint: - o python 2.x (x >= 4) + o python 2.x (x >= 6) o rpm 4.4.2.2 or newer and its python bindings o libmagic and its python bindings (optional) o readelf @@ -13,6 +13,6 @@ You need the following utilities to run rpmlint: and the following programs to install it: - o python 2.x (x >= 4) + o python 2.x (x >= 6) o rpm python bindings 4.4 or newer o sed diff --git a/InitScriptCheck.py b/InitScriptCheck.py index 9a7a4f40..116852a5 100644 --- a/InitScriptCheck.py +++ b/InitScriptCheck.py @@ -62,15 +62,15 @@ class InitScriptCheck(AbstractCheck.AbstractCheck): printError(pkg, 'init-script-name-with-dot', fname) # check chkconfig call in %post and %preun - postin = pkg[rpm.RPMTAG_POSTIN] or \ - pkg.scriptprog(rpm.RPMTAG_POSTINPROG) + postin = Pkg.b2s(pkg[rpm.RPMTAG_POSTIN]) or \ + pkg.scriptprog(rpm.RPMTAG_POSTINPROG) if not postin: printError(pkg, 'init-script-without-chkconfig-postin', fname) elif not chkconfig_regex.search(postin): printError(pkg, 'postin-without-chkconfig', fname) - preun = pkg[rpm.RPMTAG_PREUN] or \ - pkg.scriptprog(rpm.RPMTAG_PREUNPROG) + preun = Pkg.b2s(pkg[rpm.RPMTAG_PREUN]) or \ + pkg.scriptprog(rpm.RPMTAG_PREUNPROG) if not preun: printError(pkg, 'init-script-without-chkconfig-preun', fname) elif not chkconfig_regex.search(preun): diff --git a/LSBCheck.py b/LSBCheck.py index 6b06bad3..8693b832 100644 --- a/LSBCheck.py +++ b/LSBCheck.py @@ -13,6 +13,7 @@ import re import rpm from Filter import addDetails, printError +from Pkg import b2s import AbstractCheck @@ -31,15 +32,11 @@ class LSBCheck(AbstractCheck.AbstractCheck): printError(pkg, 'non-lsb-compliant-package-name', name) version = pkg[rpm.RPMTAG_VERSION] - if version: - version = version.decode() - if not version_regex.search(version): + if version and not version_regex.search(version): printError(pkg, 'non-lsb-compliant-version', version) release = pkg[rpm.RPMTAG_RELEASE] - if release: - release = release.decode() - if not version_regex.search(release): + if release and not version_regex.search(release): printError(pkg, 'non-lsb-compliant-release', release) # Create an object to enable the auto registration of the test diff --git a/MenuCheck.py b/MenuCheck.py index 8681d4e4..fdbf0a0f 100644 --- a/MenuCheck.py +++ b/MenuCheck.py @@ -199,15 +199,15 @@ class MenuCheck(AbstractCheck.AbstractCheck): printError(pkg, 'menu-in-wrong-dir', fname) if menus: - postin = pkg[rpm.RPMTAG_POSTIN] or \ - pkg.scriptprog(rpm.RPMTAG_POSTINPROG) + postin = Pkg.b2s(pkg[rpm.RPMTAG_POSTIN]) or \ + pkg.scriptprog(rpm.RPMTAG_POSTINPROG) if not postin: printError(pkg, 'menu-without-postin') elif not update_menus_regex.search(postin): printError(pkg, 'postin-without-update-menus') - postun = pkg[rpm.RPMTAG_POSTUN] or \ - pkg.scriptprog(rpm.RPMTAG_POSTUNPROG) + postun = Pkg.b2s(pkg[rpm.RPMTAG_POSTUN]) or \ + pkg.scriptprog(rpm.RPMTAG_POSTUNPROG) if not postun: printError(pkg, 'menu-without-postun') elif not update_menus_regex.search(postun): diff --git a/Pkg.py b/Pkg.py index 0e78885e..965a3911 100644 --- a/Pkg.py +++ b/Pkg.py @@ -29,14 +29,21 @@ import rpm import Filter -# Python 2/3 compatibility/convenience wrapper for printing to stderr with -# less concerns of UnicodeErrors than plain sys.stderr.write. +# warn(): 2/3 compatibility/convenience wrapper for printing to stderr with +# less concerns of UnicodeErrors than plain sys.stderr.write. +# b2s(): bytes to str if sys.version_info[0] > 2: # Blows up with Python < 3 without the exec() hack exec('def warn(s): print (s, file=sys.stderr)') long = int + def b2s(b): + if b is None: + return None + return b.decode() else: def warn(s): print >> sys.stderr, s + def b2s(b): + return b # utilities @@ -86,7 +93,7 @@ def getstatusoutput(cmd, stdoutonly = False, shell = False): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) proc.stdin.close() - text = proc.stdout.read().decode() + text = b2s(proc.stdout.read()) sts = proc.wait() if sts is None: sts = 0 @@ -445,7 +452,7 @@ class Pkg: os.close(fd) self.is_source = not self.header[rpm.RPMTAG_SOURCERPM] - self.name = self.header[rpm.RPMTAG_NAME].decode() + self.name = b2s(self.header[rpm.RPMTAG_NAME]) if self.isNoSource(): self.arch = 'nosrc' elif self.isSource(): @@ -471,6 +478,8 @@ class Pkg: if val == []: return None else: + if key in (rpm.RPMTAG_VERSION, rpm.RPMTAG_RELEASE, rpm.RPMTAG_ARCH): + val = b2s(val) return val # return the name of the directory where the package is extracted @@ -531,7 +540,7 @@ class Pkg: # LANGUAGE trumps other env vars per GNU gettext docs, see also #166 orig = os.environ.get('LANGUAGE') os.environ['LANGUAGE'] = lang - ret = self[tag].decode() + ret = b2s(self[tag]) if orig is not None: os.environ['LANGUAGE'] = orig return ret @@ -596,17 +605,17 @@ class Pkg: modes = self.header[rpm.RPMTAG_FILEMODES] users = self.header[rpm.RPMTAG_FILEUSERNAME] groups = self.header[rpm.RPMTAG_FILEGROUPNAME] - links = [x.decode() for x in self.header[rpm.RPMTAG_FILELINKTOS]] + links = [b2s(x) for x in self.header[rpm.RPMTAG_FILELINKTOS]] sizes = self.header[rpm.RPMTAG_FILESIZES] md5s = self.header[rpm.RPMTAG_FILEMD5S] mtimes = self.header[rpm.RPMTAG_FILEMTIMES] rdevs = self.header[rpm.RPMTAG_FILERDEVS] langs = self.header[rpm.RPMTAG_FILELANGS] inodes = self.header[rpm.RPMTAG_FILEINODES] - requires = [x.decode() for x in self.header[rpm.RPMTAG_FILEREQUIRE]] - provides = [x.decode() for x in self.header[rpm.RPMTAG_FILEPROVIDE]] - files = [x.decode() for x in self.header[rpm.RPMTAG_FILENAMES]] - magics = [x.decode() for x in self.header[rpm.RPMTAG_FILECLASS]] + requires = [b2s(x) for x in self.header[rpm.RPMTAG_FILEREQUIRE]] + provides = [b2s(x) for x in self.header[rpm.RPMTAG_FILEPROVIDE]] + files = [b2s(x) for x in self.header[rpm.RPMTAG_FILENAMES]] + magics = [b2s(x) for x in self.header[rpm.RPMTAG_FILECLASS]] try: # rpm >= 4.7.0 filecaps = self.header[rpm.RPMTAG_FILECAPS] except: @@ -717,8 +726,8 @@ class Pkg: if versions: for loop in range(len(versions)): - name = names[loop].decode() - evr = stringToVersion(versions[loop].decode()) + name = b2s(names[loop]) + evr = stringToVersion(b2s(versions[loop])) if prereq is not None and flags[loop] & PREREQ_FLAG: prereq.append((name, flags[loop] & (~PREREQ_FLAG), evr)) else: @@ -756,11 +765,11 @@ class Pkg: interpreter arguments, if any.""" prog = self[which] if prog is None: - prog = "" + prog = b'' elif isinstance(prog, (list, tuple)): # http://rpm.org/ticket/847#comment:2 - prog = " ".join(prog) - return prog + prog = b' '.join(prog) + return b2s(prog) def getInstalledPkgs(name): """Get list of installed package objects by name.""" diff --git a/TagsCheck.py b/TagsCheck.py index c6f5dda6..f1d12262 100644 --- a/TagsCheck.py +++ b/TagsCheck.py @@ -543,9 +543,8 @@ class TagsCheck(AbstractCheck.AbstractCheck): def check(self, pkg): - packager = pkg[rpm.RPMTAG_PACKAGER] + packager = Pkg.b2s(pkg[rpm.RPMTAG_PACKAGER]) if packager: - packager = packager.decode() self._unexpanded_macros(pkg, 'Packager', packager) if Config.getOption('Packager') and \ not packager_regex.search(packager): @@ -555,7 +554,6 @@ class TagsCheck(AbstractCheck.AbstractCheck): version = pkg[rpm.RPMTAG_VERSION] if version: - version = version.decode() self._unexpanded_macros(pkg, 'Version', version) res = invalid_version_regex.search(version) if res: @@ -565,7 +563,6 @@ class TagsCheck(AbstractCheck.AbstractCheck): release = pkg[rpm.RPMTAG_RELEASE] if release: - release = release.decode() self._unexpanded_macros(pkg, 'Release', release) if release_ext and not extension_regex.search(release): printWarning(pkg, 'not-standard-release-extension', release) @@ -684,11 +681,10 @@ class TagsCheck(AbstractCheck.AbstractCheck): langs = pkg[rpm.RPMTAG_HEADERI18NTABLE] if langs: - langs = [x.decode() for x in langs] + langs = [Pkg.b2s(x) for x in langs] - summary = pkg[rpm.RPMTAG_SUMMARY] + summary = Pkg.b2s(pkg[rpm.RPMTAG_SUMMARY]) if summary: - summary = summary.decode() if not langs: self._unexpanded_macros(pkg, 'Summary', summary) else: @@ -697,9 +693,8 @@ class TagsCheck(AbstractCheck.AbstractCheck): else: printError(pkg, 'no-summary-tag') - description = pkg[rpm.RPMTAG_DESCRIPTION] + description = Pkg.b2s(pkg[rpm.RPMTAG_DESCRIPTION]) if description: - description = description.decode() if not langs: self._unexpanded_macros(pkg, '%description', description) else: diff --git a/rpmdiff b/rpmdiff index 4135ab30..2cb23403 100755 --- a/rpmdiff +++ b/rpmdiff @@ -228,16 +228,16 @@ class Rpmdiff: if namestr == 'REQUIRES': namestr = self.req2str(oldentry[1]) self.__add(self.DEPFORMAT, - (self.REMOVED, namestr, oldentry[0].decode(), - self.sense2str(oldentry[1]), oldentry[2].decode())) + (self.REMOVED, namestr, Pkg.b2s(oldentry[0]), + self.sense2str(oldentry[1]), Pkg.b2s(oldentry[2]))) for newentry in n: if not newentry in o: namestr = name if namestr == 'REQUIRES': namestr = self.req2str(newentry[1]) self.__add(self.DEPFORMAT, - (self.ADDED, namestr, newentry[0].decode(), - self.sense2str(newentry[1]), newentry[2].decode())) + (self.ADDED, namestr, Pkg.b2s(newentry[0]), + self.sense2str(newentry[1]), Pkg.b2s(newentry[2]))) def __fileIteratorToDict(self, fi): result = {}