Further Python 3 porting

Thanks to Alberto Planas Dominguez <aplanas@suse.com> for sanity
checks and suggestions.

BinariesCheck.py

@@ -178,7 +178,7 @@ class BinaryInfo:
         try:
             fobj = open(path, 'rb')
             fobj.seek(-12, 2) # 2 == os.SEEK_END, for python 2.4 compat (#172)
-            self.tail = fobj.read().decode()
+            self.tail = Pkg.b2s(fobj.read())
         except Exception:
             e = sys.exc_info()[1]
             printWarning(pkg, 'binaryinfo-tail-failed %s: %s' % (file, e))
@@ -274,7 +274,7 @@ class BinariesCheck(AbstractCheck.AbstractCheck):
         multi_pkg = False
         srpm = pkg[rpm.RPMTAG_SOURCERPM]
         if srpm:
-            res = srcname_regex.search(srpm.decode())
+            res = srcname_regex.search(Pkg.b2s(srpm))
             if res:
                 multi_pkg = (pkg.name != res.group(1))
 

DocFilesCheck.py

@@ -17,6 +17,7 @@
 import rpm
 
 from Filter import addDetails, printWarning
+from Pkg import b2s
 import AbstractCheck
 
 
@@ -44,7 +45,7 @@ class DocFilesCheck(AbstractCheck.AbstractCheck):
 
         # register things which are provided by the package
         for i in pkg.header[rpm.RPMTAG_PROVIDES]:
-            core_reqs[i.decode()] = []
+            core_reqs[b2s(i)] = []
         for i in files:
             core_reqs[i] = []
 

FilesCheck.py

@@ -12,13 +12,12 @@ from datetime import datetime
 import os
 import re
 import stat
-import string
 import sys
 
 import rpm
 
 from Filter import addDetails, printError, printWarning
-from Pkg import catcmd, getstatusoutput, is_utf8, is_utf8_str
+from Pkg import b2s, catcmd, getstatusoutput, is_utf8, is_utf8_str
 import AbstractCheck
 import Config
 
@@ -211,7 +210,7 @@ lib_path_regex = re.compile('^(/usr(/X11R6)?)?/lib(64)?')
 lib_package_regex = re.compile('^(lib|.+-libs)')
 hidden_file_regex = re.compile('/\.[^/]*$')
 manifest_perl_regex = re.compile('^/usr/share/doc/perl-.*/MANIFEST(\.SKIP)?$')
-shebang_regex = re.compile('^#!\s*(\S+)')
+shebang_regex = re.compile(b'^#!\s*(\S+)')
 interpreter_regex = re.compile('^/(usr/)?(s?bin|games|libexec(/.+)?|(lib(64)?|share)/.+)/[^/]+$')
 script_regex = re.compile('^/((usr/)?s?bin|etc/(rc\.d/init\.d|X11/xinit\.d|cron\.(hourly|daily|monthly|weekly)))/')
 sourced_script_regex = re.compile('^/etc/(bash_completion\.d|profile\.d)/')
@@ -246,15 +245,21 @@ man_nowarn_regex = re.compile(
     r'(can\'t break|cannot adjust) line')
 man_warn_category = Config.getOption('ManWarningCategory', 'mac')
 
-fsf_license_regex = re.compile('(GNU((\s+(Library|Lesser|Affero))?(\s+General)?\s+Public|\s+Free\s+Documentation)\s+Licen[cs]e|(GP|FD)L)', re.IGNORECASE)
+fsf_license_regex = re.compile(b'(GNU((\s+(Library|Lesser|Affero))?(\s+General)?\s+Public|\s+Free\s+Documentation)\s+Licen[cs]e|(GP|FD)L)', re.IGNORECASE)
-fsf_wrong_address_regex = re.compile('(675\s+Mass\s+Ave|59\s+Temple\s+Place|Franklin\s+Steet|02139|02111-1307)', re.IGNORECASE)
+fsf_wrong_address_regex = re.compile(b'(675\s+Mass\s+Ave|59\s+Temple\s+Place|Franklin\s+Steet|02139|02111-1307)', re.IGNORECASE)
 
 scalable_icon_regex = re.compile(r'^/usr(?:/local)?/share/icons/.*/scalable/')
 
-# loosely inspired from Python Cookbook
+# "is binary" stuff borrowed from https://pypi.python.org/pypi/binaryornot
-# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/173220
+# TODO: switch to it sometime later instead of embedding our own copy
-text_characters = "".join(map(chr, range(32, 127))) + "\n\r\t\b"
+
-_null_trans = string.maketrans("", "")
+printable_extended_ascii = b'\n\r\t\f\b'
+if bytes is str:
+    # Python 2 means we need to invoke chr() explicitly
+    printable_extended_ascii += b''.join(map(chr, range(32, 256)))
+else:
+    # Python 3 means bytes accepts integer input directly
+    printable_extended_ascii += bytes(range(32, 256))
 
 def peek(filename, pkg, length=1024):
     """Peek into a file, return a chunk from its beginning and a flag if it
@@ -272,7 +277,7 @@ def peek(filename, pkg, length=1024):
             fobj.close()
         return (chunk, False)
 
-    if "\0" in chunk:
+    if b'\0' in chunk:
         return (chunk, False)
 
     if not chunk:  # Empty files are considered text
@@ -281,18 +286,17 @@ def peek(filename, pkg, length=1024):
     fl = filename.lower()
 
     # PDF's are binary but often detected as text by the algorithm below
-    if fl.endswith('.pdf') and chunk.startswith('%PDF-'):
+    if fl.endswith('.pdf') and chunk.startswith(b'%PDF-'):
         return (chunk, False)
     # Ditto RDoc RI files
     if fl.endswith('.ri') and '/ri/' in fl:
         return (chunk, False)
 
-    # Get the non-text characters (maps a character to itself then
+    # Binary if control chars are > 30% of the string
-    # use the 'remove' option to get rid of the text characters.)
+    control_chars = chunk.translate(None, printable_extended_ascii)
-    t = chunk.translate(_null_trans, text_characters)
+    nontext_ratio = float(len(control_chars)) / float(len(chunk))
+    istext = nontext_ratio <= 0.30
 
-    # If more than 30% non-text characters, then consider it a binary file
-    istext = float(len(t))/len(chunk) <= 0.30
     return (chunk, istext)
 
 # See Python sources for a full list of the values here.
@@ -523,12 +527,12 @@ class FilesCheck(AbstractCheck.AbstractCheck):
                                    oct(perm))
 
                 # Prefetch scriptlets, strip quotes from them (#169)
-                postin = (pkg[rpm.RPMTAG_POSTIN] or \
+                postin = b2s(pkg[rpm.RPMTAG_POSTIN]) or \
-                    pkg.scriptprog(rpm.RPMTAG_POSTINPROG)).decode()
+                         pkg.scriptprog(rpm.RPMTAG_POSTINPROG)
                 if postin:
                     postin = quotes_regex.sub('', postin)
-                postun = (pkg[rpm.RPMTAG_POSTUN] or \
+                postun = b2s(pkg[rpm.RPMTAG_POSTUN]) or \
-                    pkg.scriptprog(rpm.RPMTAG_POSTUNPROG)).decode()
+                         pkg.scriptprog(rpm.RPMTAG_POSTUNPROG)
                 if postun:
                     postun = quotes_regex.sub('', postun)
 
@@ -556,7 +560,7 @@ class FilesCheck(AbstractCheck.AbstractCheck):
                 if chunk:
                     res = shebang_regex.search(chunk)
                     if res:
-                        interpreter = res.group(1)
+                        interpreter = b2s(res.group(1))
 
                 if doc_regex.search(f):
                     if not interpreter:
@@ -608,8 +612,8 @@ class FilesCheck(AbstractCheck.AbstractCheck):
                     elif not install_info_regex.search(postin):
                         printError(pkg, 'postin-without-install-info', f)
 
-                    preun = pkg[rpm.RPMTAG_PREUN] or \
+                    preun = b2s(pkg[rpm.RPMTAG_PREUN]) or \
-                        pkg.scriptprog(rpm.RPMTAG_PREUNPROG)
+                            pkg.scriptprog(rpm.RPMTAG_PREUNPROG)
                     if not postun and not preun:
                         printError(pkg,
                                    'info-files-without-install-info-postun', f)
@@ -815,11 +819,11 @@ class FilesCheck(AbstractCheck.AbstractCheck):
                         if not mode_is_exec and not is_doc:
                             printError(pkg, 'non-executable-script', f,
                                        oct(perm), interpreter)
-                        if '\r' in chunk:
+                        if b'\r' in chunk:
                             printError(
                                 pkg, 'wrong-script-end-of-line-encoding', f)
                     elif is_doc and not skipdocs_regex.search(f):
-                        if '\r' in chunk:
+                        if b'\r' in chunk:
                             printWarning(
                                 pkg, 'wrong-file-end-of-line-encoding', f)
                         # We check only doc text files for UTF-8-ness;

INSTALL

@@ -1,6 +1,6 @@
 You need the following utilities to run rpmlint:
 
-	o python 2.x (x >= 4)
+	o python 2.x (x >= 6)
 	o rpm 4.4.2.2 or newer and its python bindings
 	o libmagic and its python bindings (optional)
 	o readelf
@@ -13,6 +13,6 @@ You need the following utilities to run rpmlint:
 
 and the following programs to install it:
 
-	o python 2.x (x >= 4)
+	o python 2.x (x >= 6)
 	o rpm python bindings 4.4 or newer
 	o sed

InitScriptCheck.py

@@ -62,15 +62,15 @@ class InitScriptCheck(AbstractCheck.AbstractCheck):
                 printError(pkg, 'init-script-name-with-dot', fname)
 
             # check chkconfig call in %post and %preun
-            postin = pkg[rpm.RPMTAG_POSTIN] or \
+            postin = Pkg.b2s(pkg[rpm.RPMTAG_POSTIN]) or \
-                pkg.scriptprog(rpm.RPMTAG_POSTINPROG)
+                     pkg.scriptprog(rpm.RPMTAG_POSTINPROG)
             if not postin:
                 printError(pkg, 'init-script-without-chkconfig-postin', fname)
             elif not chkconfig_regex.search(postin):
                 printError(pkg, 'postin-without-chkconfig', fname)
 
-            preun = pkg[rpm.RPMTAG_PREUN] or \
+            preun = Pkg.b2s(pkg[rpm.RPMTAG_PREUN]) or \
-                pkg.scriptprog(rpm.RPMTAG_PREUNPROG)
+                    pkg.scriptprog(rpm.RPMTAG_PREUNPROG)
             if not preun:
                 printError(pkg, 'init-script-without-chkconfig-preun', fname)
             elif not chkconfig_regex.search(preun):

LSBCheck.py

@@ -13,6 +13,7 @@ import re
 import rpm
 
 from Filter import addDetails, printError
+from Pkg import b2s
 import AbstractCheck
 
 
@@ -31,15 +32,11 @@ class LSBCheck(AbstractCheck.AbstractCheck):
             printError(pkg, 'non-lsb-compliant-package-name', name)
 
         version = pkg[rpm.RPMTAG_VERSION]
-        if version:
+        if version and not version_regex.search(version):
-            version = version.decode()
-            if not version_regex.search(version):
                 printError(pkg, 'non-lsb-compliant-version', version)
 
         release = pkg[rpm.RPMTAG_RELEASE]
-        if release:
+        if release and not version_regex.search(release):
-            release = release.decode()
-            if not version_regex.search(release):
                 printError(pkg, 'non-lsb-compliant-release', release)
 
 # Create an object to enable the auto registration of the test

MenuCheck.py

@@ -199,15 +199,15 @@ class MenuCheck(AbstractCheck.AbstractCheck):
                     printError(pkg, 'menu-in-wrong-dir', fname)
 
         if menus:
-            postin = pkg[rpm.RPMTAG_POSTIN] or \
+            postin = Pkg.b2s(pkg[rpm.RPMTAG_POSTIN]) or \
-                pkg.scriptprog(rpm.RPMTAG_POSTINPROG)
+                     pkg.scriptprog(rpm.RPMTAG_POSTINPROG)
             if not postin:
                 printError(pkg, 'menu-without-postin')
             elif not update_menus_regex.search(postin):
                 printError(pkg, 'postin-without-update-menus')
 
-            postun = pkg[rpm.RPMTAG_POSTUN] or \
+            postun = Pkg.b2s(pkg[rpm.RPMTAG_POSTUN]) or \
-                pkg.scriptprog(rpm.RPMTAG_POSTUNPROG)
+                     pkg.scriptprog(rpm.RPMTAG_POSTUNPROG)
             if not postun:
                 printError(pkg, 'menu-without-postun')
             elif not update_menus_regex.search(postun):

Pkg.py

@@ -29,14 +29,21 @@ import rpm
 
 import Filter
 
-# Python 2/3 compatibility/convenience wrapper for printing to stderr with
+# warn(): 2/3 compatibility/convenience wrapper for printing to stderr with
-# less concerns of UnicodeErrors than plain sys.stderr.write.
+#         less concerns of UnicodeErrors than plain sys.stderr.write.
+# b2s():  bytes to str
 if sys.version_info[0] > 2:
     # Blows up with Python < 3 without the exec() hack
     exec('def warn(s): print (s, file=sys.stderr)')
     long = int
+    def b2s(b):
+        if b is None:
+            return None
+        return b.decode()
 else:
     def warn(s): print >> sys.stderr, s
+    def b2s(b):
+        return b
 
 
 # utilities
@@ -86,7 +93,7 @@ def getstatusoutput(cmd, stdoutonly = False, shell = False):
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT, close_fds=True)
     proc.stdin.close()
-    text = proc.stdout.read().decode()
+    text = b2s(proc.stdout.read())
     sts = proc.wait()
     if sts is None:
         sts = 0
@@ -445,7 +452,7 @@ class Pkg:
                 os.close(fd)
             self.is_source = not self.header[rpm.RPMTAG_SOURCERPM]
 
-        self.name = self.header[rpm.RPMTAG_NAME].decode()
+        self.name = b2s(self.header[rpm.RPMTAG_NAME])
         if self.isNoSource():
             self.arch = 'nosrc'
         elif self.isSource():
@@ -471,6 +478,8 @@ class Pkg:
         if val == []:
             return None
         else:
+            if key in (rpm.RPMTAG_VERSION, rpm.RPMTAG_RELEASE, rpm.RPMTAG_ARCH):
+                val = b2s(val)
             return val
 
     # return the name of the directory where the package is extracted
@@ -531,7 +540,7 @@ class Pkg:
         # LANGUAGE trumps other env vars per GNU gettext docs, see also #166
         orig = os.environ.get('LANGUAGE')
         os.environ['LANGUAGE'] = lang
-        ret = self[tag].decode()
+        ret = b2s(self[tag])
         if orig is not None:
             os.environ['LANGUAGE'] = orig
         return ret
@@ -596,17 +605,17 @@ class Pkg:
         modes = self.header[rpm.RPMTAG_FILEMODES]
         users = self.header[rpm.RPMTAG_FILEUSERNAME]
         groups = self.header[rpm.RPMTAG_FILEGROUPNAME]
-        links = [x.decode() for x in self.header[rpm.RPMTAG_FILELINKTOS]]
+        links = [b2s(x) for x in self.header[rpm.RPMTAG_FILELINKTOS]]
         sizes = self.header[rpm.RPMTAG_FILESIZES]
         md5s = self.header[rpm.RPMTAG_FILEMD5S]
         mtimes = self.header[rpm.RPMTAG_FILEMTIMES]
         rdevs = self.header[rpm.RPMTAG_FILERDEVS]
         langs = self.header[rpm.RPMTAG_FILELANGS]
         inodes = self.header[rpm.RPMTAG_FILEINODES]
-        requires = [x.decode() for x in self.header[rpm.RPMTAG_FILEREQUIRE]]
+        requires = [b2s(x) for x in self.header[rpm.RPMTAG_FILEREQUIRE]]
-        provides = [x.decode() for x in self.header[rpm.RPMTAG_FILEPROVIDE]]
+        provides = [b2s(x) for x in self.header[rpm.RPMTAG_FILEPROVIDE]]
-        files = [x.decode() for x in self.header[rpm.RPMTAG_FILENAMES]]
+        files = [b2s(x) for x in self.header[rpm.RPMTAG_FILENAMES]]
-        magics = [x.decode() for x in self.header[rpm.RPMTAG_FILECLASS]]
+        magics = [b2s(x) for x in self.header[rpm.RPMTAG_FILECLASS]]
         try: # rpm >= 4.7.0
             filecaps = self.header[rpm.RPMTAG_FILECAPS]
         except:
@@ -717,8 +726,8 @@ class Pkg:
 
         if versions:
             for loop in range(len(versions)):
-                name = names[loop].decode()
+                name = b2s(names[loop])
-                evr = stringToVersion(versions[loop].decode())
+                evr = stringToVersion(b2s(versions[loop]))
                 if prereq is not None and flags[loop] & PREREQ_FLAG:
                     prereq.append((name, flags[loop] & (~PREREQ_FLAG), evr))
                 else:
@@ -756,11 +765,11 @@ class Pkg:
            interpreter arguments, if any."""
         prog = self[which]
         if prog is None:
-            prog = ""
+            prog = b''
         elif isinstance(prog, (list, tuple)):
             # http://rpm.org/ticket/847#comment:2
-            prog = " ".join(prog)
+            prog = b' '.join(prog)
-        return prog
+        return b2s(prog)
 
 def getInstalledPkgs(name):
     """Get list of installed package objects by name."""

TagsCheck.py

@@ -543,9 +543,8 @@ class TagsCheck(AbstractCheck.AbstractCheck):
 
     def check(self, pkg):
 
-        packager = pkg[rpm.RPMTAG_PACKAGER]
+        packager = Pkg.b2s(pkg[rpm.RPMTAG_PACKAGER])
         if packager:
-            packager = packager.decode()
             self._unexpanded_macros(pkg, 'Packager', packager)
             if Config.getOption('Packager') and \
                not packager_regex.search(packager):
@@ -555,7 +554,6 @@ class TagsCheck(AbstractCheck.AbstractCheck):
 
         version = pkg[rpm.RPMTAG_VERSION]
         if version:
-            version = version.decode()
             self._unexpanded_macros(pkg, 'Version', version)
             res = invalid_version_regex.search(version)
             if res:
@@ -565,7 +563,6 @@ class TagsCheck(AbstractCheck.AbstractCheck):
 
         release = pkg[rpm.RPMTAG_RELEASE]
         if release:
-            release = release.decode()
             self._unexpanded_macros(pkg, 'Release', release)
             if release_ext and not extension_regex.search(release):
                 printWarning(pkg, 'not-standard-release-extension', release)
@@ -684,11 +681,10 @@ class TagsCheck(AbstractCheck.AbstractCheck):
 
         langs = pkg[rpm.RPMTAG_HEADERI18NTABLE]
         if langs:
-            langs = [x.decode() for x in langs]
+            langs = [Pkg.b2s(x) for x in langs]
 
-        summary = pkg[rpm.RPMTAG_SUMMARY]
+        summary = Pkg.b2s(pkg[rpm.RPMTAG_SUMMARY])
         if summary:
-            summary = summary.decode()
             if not langs:
                 self._unexpanded_macros(pkg, 'Summary', summary)
             else:
@@ -697,9 +693,8 @@ class TagsCheck(AbstractCheck.AbstractCheck):
         else:
             printError(pkg, 'no-summary-tag')
 
-        description = pkg[rpm.RPMTAG_DESCRIPTION]
+        description = Pkg.b2s(pkg[rpm.RPMTAG_DESCRIPTION])
         if description:
-            description = description.decode()
             if not langs:
                 self._unexpanded_macros(pkg, '%description', description)
             else:

rpmdiff

@@ -228,16 +228,16 @@ class Rpmdiff:
                 if namestr == 'REQUIRES':
                     namestr = self.req2str(oldentry[1])
                 self.__add(self.DEPFORMAT,
-                           (self.REMOVED, namestr, oldentry[0].decode(),
+                           (self.REMOVED, namestr, Pkg.b2s(oldentry[0]),
-                            self.sense2str(oldentry[1]), oldentry[2].decode()))
+                            self.sense2str(oldentry[1]), Pkg.b2s(oldentry[2])))
         for newentry in n:
             if not newentry in o:
                 namestr = name
                 if namestr == 'REQUIRES':
                     namestr = self.req2str(newentry[1])
                 self.__add(self.DEPFORMAT,
-                           (self.ADDED, namestr, newentry[0].decode(),
+                           (self.ADDED, namestr, Pkg.b2s(newentry[0]),
-                            self.sense2str(newentry[1]), newentry[2].decode()))
+                            self.sense2str(newentry[1]), Pkg.b2s(newentry[2])))
 
     def __fileIteratorToDict(self, fi):
         result = {}