Merge "Add script to verify that repo's SRPMs matches with mirror's SRPMs"

This commit is contained in:
Zuul 2018-07-20 04:14:52 +00:00 committed by Gerrit Code Review
commit 5ee339a7e0
1 changed files with 214 additions and 0 deletions

214
DependenciesReviewer.py Normal file
View File

@ -0,0 +1,214 @@
import os
import subprocess
import getpass
import sys
# Error codes
SUCCESS = 0
RPMMISMATCH = 1
FILENOTFOUND = 2
ERRORCODE = SUCCESS
# Log file
results = open("dependenciesreviewer.log", "a")
# Parametrized variables
SELECT = {"centos" : {"name":"centos",
"dependencies":"srpm_path",
"prefix": "rpms_"
},
}
DISTRO = SELECT["centos"]
# Global variables
USER = getpass.getuser()
WORK = os.path.abspath("..")
REPOS = os.path.join(WORK, "cgcs-root/stx/")
MTOOLS = os.path.join(WORK, "stx-tools/centos-mirror-tools")
class PkgInfo:
"""
PkgInfo has a name, mirror location and full path
"""
def __init__(self, name, location="NotFound", fullpath=""):
"""
Initialize PkgInfo object
"""
self.name = name
self.location = location
self.fullpath = fullpath
def __str__(self):
"""
Return information about PkgInfo object
"""
return "Name: {}\nLocation: {}\nFull Path: {}\n".format(self.name,
self.location,
self.fullpath)
def print_with_comment_if_not_found(self, comment=""):
"""
Prints the full path of an PkgInfo object if it doesn't have
location, followed by a comment provided by the user.
"""
if self.location == "NotFound":
print(">>> {} {}".format(self.fullpath, comment), file=results)
class MirrorInfo:
"""
MirrorInfo has a path and a list of source packages
"""
def __init__(self, path, src_pkgs=None):
"""
Initialize MirrorInfo object
"""
self.path = path
self.src_pkgs = src_pkgs
def __str__(self):
"""
Return information about MirrorInfo object
"""
return "MirrorInfo: {} {}".format(self.path, self.src_pkgs)
class DependenciesReviewer:
"""
DependenciesReviewer class reviews the content in stx-'s
*/centos/srpm_path matches with the information in the mirror's lists.
If there are modules that does not match, the DependenciesReviewer can
display the information.
"""
def __init__(self, modulepath=os.path.abspath(".."),
mirrorpath=os.path.abspath(".")):
self.modulepath = modulepath
self.mirrorpath = mirrorpath
self._src_pkgs_dict = {}
self._src_pkgs_list = []
def __str__(self):
return "DependenciesReviewer: {} {} {}".format(self.modulepath,
self.mirrorpath)
def _find_elements(self, spkgsdict, mirror_list, mirror_path):
"""
Fill the dictionary with the location in the mirror's list
"""
for key, value in spkgsdict.items():
for i in range(0, len(value)):
if value[i].name in mirror_list:
spkgsdict[key][i].location = mirror_path
return spkgsdict
def _get_content(self, path):
""" Get path's content as a list """
try:
text = open(path).read()
text_list = text.split("\n")
text_list = list(filter(None, text_list))
return text_list
except FileNotFoundError:
print("Mirror lst file not found {}".format(path.split("/")[-1]),
file=results)
ERRORCODE = FILENOTFOUND
return []
def check_missing(self):
"""
Solve the dependencies
"""
# SOURCE PACKAGES
# Get the paths for all source packages information files in the repo
vardir = os.path.join("*", DISTRO["name"], DISTRO["dependencies"])
packages_paths = subprocess.check_output(['find',
self.modulepath,
'-wholename',
vardir])
packages_paths = packages_paths.decode('utf-8')
packages_paths = packages_paths.split("\n")
packages_paths = list(filter(None, packages_paths))
# Fill dictionary and list with the content from those files
for path in packages_paths:
pkgs_list = open(path).read()
pkgs_list = pkgs_list.split("\n")
pkgs_list = list(filter(None, pkgs_list))
if not pkgs_list:
print("No content in: "+path, file=results)
else:
temp = []
for pkg in pkgs_list:
if "mirror:" in pkg:
pkgname = pkg.split("/")[-1]
temp.append(PkgInfo(pkgname, location="NotFound",
fullpath=pkg))
self._src_pkgs_list.append(pkgname)
self._src_pkgs_dict[path] = temp
# MIRROR LISTS
# Generate list of MirrorInfo objects, which is needed for the review
all_files = os.listdir(self.mirrorpath)
mirror_files = [x for x in all_files if DISTRO["prefix"] in x]
_spkg_mirror = []
for elem in mirror_files:
# Get package's content
_tmp_path = os.path.join(self.mirrorpath, elem)
_tmp_pkgs = self._get_content(_tmp_path)
# Do particular clean up for 3rd party packages' names
if elem == DISTRO["prefix"]+"from_3rd_parties.lst":
_tmp_pkgs = [x.split("#")[0] for x in _tmp_pkgs]
# Create a list with the Mirror Info
_spkg_mirror.append(MirrorInfo(path=_tmp_path, src_pkgs=_tmp_pkgs))
# MATCHING
# Finding Packages in the mirror
for mirr in _spkg_mirror:
# Fill the dictinoary with the location
self._src_pkgs_dict = self._find_elements(self._src_pkgs_dict,
mirr.src_pkgs,
mirr.path)
# Leave on the list only the missing Source Packages
self._src_pkgs_list = [element for element in self._src_pkgs_list
if element not in mirr.src_pkgs]
def how_many_missing(self):
"""
Return the number of missing RPMs
"""
return len(self._src_pkgs_list)
def show_missing(self):
"""
Show the DependenciesReviewer results based on how it was initialized
"""
for key, value in self._src_pkgs_dict.items():
for val in value:
val.print_with_comment_if_not_found(key)
if __name__ == "__main__":
try:
directories = os.listdir(REPOS)
except FileNotFoundError:
print("Directory not found {}".format(REPOS), file=results)
ERRORCODE = FILENOTFOUND
if ERRORCODE == SUCCESS:
stx_directories = []
for directory in directories:
if "stx-" in directory:
A = DependenciesReviewer(modulepath=os.path.join(REPOS, directory),
mirrorpath=MTOOLS)
A.check_missing()
if A.how_many_missing() > 0:
print("Missing Src Packages in module: "+directory,
file=results)
A.show_missing()
if ERRORCODE != FILENOTFOUND:
ERRORCODE = RPMMISMATCH
else:
continue
if ERRORCODE == SUCCESS:
print("All Src Packages in stx-* repos were found in Mirror's lists.",
file=results)
results.close()
sys.exit(ERRORCODE)