Merge "Add user application generation tool"

This commit is contained in:
Zuul 2020-04-28 06:00:51 +00:00 committed by Gerrit Code Review
commit 451088b757
7 changed files with 787 additions and 0 deletions

60
app-gen-tool/README.md Normal file
View File

@ -0,0 +1,60 @@
# StarlingX Application Generation Tool
The purpose of this tool is to generate StarlingX user applications in an easy
way without stx build environment and armada manifest schema knowledge.
## Pre-requisite
1. Helm2 installed
2. python3.5+
3. pyyaml>=5.0.0 package
`$ pip3 install pyyaml==5.1.2`
## 3 Steps to create a starlingx user app
#### 1. Prepare a helm chart(s)
##### What is helm and helm chart?
Helm is a Kubernetes package and operations manager. A Helm chart can contain
any number of Kubernetes objects, all of which are deployed as part of the
chart.
A list of official Helm Charts locates [here](https://github.com/helm/charts)
##### How to develop a helm chart?
Refer to official [helm doc](https://helm.sh/docs/)
#### 2. Create an app manifest
A few essential fields needed to create the app, simplest one could be:
```
appName: stx-app
namespace: stx-app
version: 1.0-1
chart:
- name: chart1
path: /path/to/chart1
chartGroup:
- name: chartgroup1
description: "This is the first chartgroup"
sequenced: true
chart_group:
- chart1
manifest:
name: stx-app-manifest
releasePrefix: myprefix
chart_groups:
- chartgroup1
```
For more details, please refer to example.yaml
#### 3. Run app-gen.py
`$ python3 app-gen.py -i app_manifest.yaml [-o ./output] [--overwrite]`
The application will be generated automatically along with the tarball located
in the folder of your application name.

612
app-gen-tool/app-gen.py Normal file
View File

@ -0,0 +1,612 @@
import yaml
import os
import sys, getopt, getpass
import subprocess
import hashlib
import tarfile
import re
import shutil
from urllib import request
SCHEMA_CHART_TEMPLATE = 'template/armada-chart.template'
SCHEMA_CHARTGROUP_TEMPLATE = 'template/armada-chartgroup.template'
SCHEMA_MANIFEST_TEMPLATE = 'template/armada-manifest.template'
BIN_FETCH_CHART_INFO = 'bin/fetch_chart_info.sh'
TEMP_USER_DIR = '/tmp/' + getpass.getuser() + '/'
# Temp app work dir to hold git repo and upstream tarball
# TEMP_APP_DIR = TEMP_USER_DIR/appName
TEMP_APP_DIR = ''
APP_GEN_PY_PATH = os.path.split(os.path.realpath(__file__))[0]
def to_camel_case(s):
return s[0].lower() + s.title().replace('_','')[1:] if s else s
class ArmadaApplication:
def __init__(self, app_data):
# Initialize application config
self._armada_app = {}
# 'appName', 'namespace', 'version' are checked in check_manifest()
self._armada_app['appName'] = app_data['appName']
self._armada_app['namespace'] = app_data['namespace']
self._armada_app['version'] = app_data['version']
# Initialize manifest
self._armada_manifest = app_data['manifest']
# Initialize chartgroup
self._armada_chartgroup = app_data['chartGroup']
# Initialize chart
self._armada_chart = app_data['chart']
# add namespace and prefix to each chart
# 'namespace', 'releasePrefix' are checked in check_manifest()
for i in range(len(self._armada_chart)):
self._armada_chart[i]['namespace'] = self._armada_app['namespace']
self._armada_chart[i]['releasePrefix'] = self._armada_manifest['releasePrefix']
# TODO: Validate values
def _validate_app_values(self, app_data):
return True
# TODO: Validate values
def _validate_manifest_values(self, manifest_data):
return True
# TODO: Validate values
def _validate_chartgroup_values(self, chartgroup_data):
return True
# TODO: Validate values
def _validate_chart_values(self, chart_data):
return True
def _validate_app_attributes(self):
if not self._validate_app_values(self._armada_app):
return False
if not self._validate_manifest_values(self._armada_manifest):
return False
if not self._validate_chartgroup_values(self._armada_chartgroup):
return False
if not self._validate_chart_values(self._armada_chart):
return False
return True
def get_app_name(self):
return self._armada_app['appName']
def _package_helm_chart(self, chart):
path = chart['path']
# lint helm chart
cmd_lint = ['helm', 'lint', path]
subproc = subprocess.run(cmd_lint, env=os.environ.copy(), \
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if subproc.returncode == 0:
print(str(subproc.stdout, encoding = 'utf-8'))
else:
print(str(subproc.stderr, encoding = 'utf-8'))
return False
# package helm chart
cmd_package = ['helm', 'package', path, '--save=false', \
'--destination=' + self._armada_app['outputChartDir']]
subproc = subprocess.run(cmd_package, env=os.environ.copy(), \
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if subproc.returncode == 0:
output = str(subproc.stdout, encoding = 'utf-8')
print(output)
# capture tarball name
for words in output.split('/'):
if 'tgz' in words:
chart['tarballName'] = words.rstrip('\n')
else:
print(subproc.stderr)
return False
return True
# pyyaml does not support writing yaml block with initial indent
# add initial indent for yaml block substitution
def _write_yaml_to_manifest(self, key, src, init_indent):
target = {}
# add heading key
target[key] = src
lines = yaml.safe_dump(target).split('\n')
# remove ending space
lines.pop()
indents = ' ' * init_indent
for i in range(len(lines)):
lines[i] = indents + lines[i]
# restore ending '\n'
return '\n'.join(lines) + '\n'
def _substitute_values(self, in_line, dicts):
out_line = in_line
pattern = re.compile('\$.+?\$')
results = pattern.findall(out_line)
if results:
for result in results:
result_word = result.strip('$').split('%')
value_key = result_word[0]
value_default = ''
if len(result_word) > 1:
value_default = result_word[1]
# underscore case to camel case
value = to_camel_case(value_key)
if value in dicts:
out_line = out_line.replace(result, str(dicts[value]))
elif value_default:
out_line = out_line.replace(result, value_default)
if out_line == in_line:
return out_line, False
else:
return out_line, True
def _substitute_blocks(self, in_line, dicts):
out_line = in_line
result = re.search('@\S+\|\d+@',out_line)
if result:
block_key = result.group().strip('@').split('|')
key = block_key[0].lower()
indent = int(block_key[1])
if key in dicts:
out_line = self._write_yaml_to_manifest(key, dicts[key], indent)
else:
out_line = ''
return out_line
# Fetch info from helm chart to fill
# the values that needs to be substituted
# Below info are needed:
# - waitLabelKey
# - chartArcname
#
def _fetch_info_from_chart(self, chart_idx):
a_chart = self._armada_chart[chart_idx]
bin_fetch_script = APP_GEN_PY_PATH + '/' + BIN_FETCH_CHART_INFO
# waitLabelKey
# search for the key of label which indicates '.Release.Name'
# within deployment, statefulset, daemonset yaml file
cmd = [bin_fetch_script, 'waitlabel', a_chart['path']]
subproc = subprocess.run(cmd, env=os.environ.copy(), \
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if subproc.returncode == 0:
output = str(subproc.stdout, encoding = 'utf-8')
if output.strip():
a_chart['waitLabelKey'] = output.strip()
if 'waitLabelKey' not in a_chart:
print("The label which indicates .Release.Name is not found in %s" % a_chart['name'])
return False
# chartArcname is the helm chart name in Chart.yaml
# it is used as tarball arcname during helm package
cmd = [bin_fetch_script, 'chartname', a_chart['path']]
subproc = subprocess.run(cmd, env=os.environ.copy(), \
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if subproc.returncode == 0:
output = str(subproc.stdout, encoding = 'utf-8')
if output.strip():
a_chart['chartArcname'] = output.strip()
if 'chartArcname' not in a_chart:
print("The name within Chart.yaml of chart %s folder is not found" % a_chart['name'])
return False
return True
# Sub-process of app generation
# lint and package helm chart
# TODO: sub-chart dependency check
#
def _gen_helm_chart_tarball(self, chart):
ret = False
path = ''
print('Processing chart %s...' % chart['name'])
# check pathtype of the chart
if chart['_pathType'] is 'git':
gitname = ''
# download git
if not os.path.exists(TEMP_APP_DIR):
os.makedirs(TEMP_APP_DIR)
# if the git folder exists, check git name and use that folder
# otherwise git clone from upstream
if not os.path.exists(TEMP_APP_DIR + chart['_gitname']):
saved_pwd = os.getcwd()
os.chdir(TEMP_APP_DIR)
cmd = ['git', 'clone', chart['path']]
subproc = subprocess.run(cmd, env=os.environ.copy(), \
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if subproc.returncode != 0:
output = str(subproc.stderr, encoding = 'utf-8')
print(output)
print('Error: git clone %s failed' % chart['_gitname'])
os.chdir(saved_pwd)
return False
os.chdir(saved_pwd)
else:
# git pull to ensure folder up-to-date
saved_pwd = os.getcwd()
os.chdir(TEMP_APP_DIR + chart['_gitname'])
cmd = ['git', 'pull']
subproc = subprocess.run(cmd, env=os.environ.copy(), \
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if subproc.returncode != 0:
output = str(subproc.stderr, encoding = 'utf-8')
print(output)
print('Error: git pull for %s failed' % chart['_gitname'])
os.chdir(saved_pwd)
return False
os.chdir(saved_pwd)
path = TEMP_APP_DIR + chart['_gitname'] + '/' + chart['subpath']
elif chart['_pathType'] is 'tarball':
if not os.path.exists(TEMP_APP_DIR):
os.makedirs(TEMP_APP_DIR)
try:
# check whether it's a url or local tarball
if not os.path.exists(chart['path']):
# download tarball
tarpath = TEMP_APP_DIR + chart['_tarname'] + '.tgz'
if not os.path.exists(tarpath):
res = request.urlopen(chart['path'])
with open(tarpath, 'wb') as f:
f.write(res.read())
else:
tarpath = chart['path']
# extract tarball
chart_tar = tarfile.open(tarpath, 'r:gz')
chart_files = chart_tar.getnames()
# get tar arcname for packaging helm chart process
# TODO: compatible with the case that there is no arcname
chart['_tarArcname'] = chart_files[0].split('/')[0]
if not os.path.exists(chart['_tarArcname']):
for chart_file in chart_files:
chart_tar.extract(chart_file, TEMP_APP_DIR)
chart_tar.close()
except Exception as e:
print('Error: %s' % e)
return False
path = TEMP_APP_DIR + chart['_tarArcname'] + '/' + chart['subpath']
elif chart['_pathType'] is 'dir':
path = chart['path']
# update chart path
# remove ending '/'
chart['path'] = path.rstrip('/')
# lint and package
ret = self._package_helm_chart(chart)
return ret
# Sub-process of app generation
# generate application manifest file
#
def _gen_armada_manifest(self):
# check manifest file existance
manifest_file = self._armada_app['outputDir'] + '/' + self._armada_app['appName'] + '.yaml'
if os.path.exists(manifest_file):
os.remove(manifest_file)
# update schema path to abspath
chart_template = APP_GEN_PY_PATH + '/' + SCHEMA_CHART_TEMPLATE
chartgroup_template = APP_GEN_PY_PATH + '/' + SCHEMA_CHARTGROUP_TEMPLATE
manifest_template = APP_GEN_PY_PATH + '/' + SCHEMA_MANIFEST_TEMPLATE
# generate chart schema
try:
with open(chart_template, 'r') as f:
chart_schema = f.readlines()
except IOError:
print('File %s not found' % chart_template)
return False
with open(manifest_file, 'a') as f:
# iterate each armada_chart
for idx in range(len(self._armada_chart)):
a_chart = self._armada_chart[idx]
# fetch chart specific info
if not self._fetch_info_from_chart(idx):
return False
for line in chart_schema:
# substitute template values to chart values
out_line, substituted = self._substitute_values(line, a_chart)
if not substituted:
# substitute template blocks to chart blocks
out_line = self._substitute_blocks(line, a_chart)
f.write(out_line)
# generate chartgroup schema
try:
with open(chartgroup_template, 'r') as f:
chartgroup_schema = f.readlines()
except IOError:
print('File %s not found' % chartgroup_template)
return False
with open(manifest_file, 'a') as f:
# iterate each chartgroup
for chartgroup in self._armada_chartgroup:
for line in chartgroup_schema:
# substitute template values to chartgroup values
out_line, substituted = self._substitute_values(line, chartgroup)
if not substituted:
# substitute template blocks to chartgroup blocks
out_line = self._substitute_blocks(line, chartgroup)
f.write(out_line)
# generate manifest schema
try:
with open(manifest_template, 'r') as f:
manifest_schema = f.readlines()
except IOError:
print('File %s not found' % manifest_template)
return False
with open(manifest_file, 'a') as f:
# only one manifest in an application
manifest = self._armada_manifest
# substitute values
for line in manifest_schema:
# substitute template values to manifest values
out_line, substituted = self._substitute_values(line, manifest)
if not substituted:
# substitute template blocks to manifest blocks
out_line = self._substitute_blocks(line, manifest)
f.write(out_line)
return True
# Sub-process of app generation
# generate application metadata
#
def _gen_metadata(self):
# check metadata file existance
metadata_file = self._armada_app['outputDir'] + '/metadata.yaml'
if os.path.exists(metadata_file):
os.remove(metadata_file)
with open(metadata_file, 'a') as f:
f.write('app_name: ' + self._armada_app['appName'] + '\n')
f.write('app_version: ' + self._armada_app['version'] + '\n')
return True
def _gen_md5(self, in_file):
with open(in_file, 'rb') as f:
out_md5 = hashlib.md5(f.read()).hexdigest()
return out_md5
# Sub-process of app generation
# generate application checksum file and tarball
#
def _gen_checksum_and_app_tarball(self):
store_cwd = os.getcwd()
os.chdir(self._armada_app['outputDir'])
# gen checksum
# check checksum file existance
checksum_file = 'checksum.md5'
if os.path.exists(checksum_file):
os.remove(checksum_file)
app_files = []
for parent, dirnames, filenames in os.walk('./'):
for filename in filenames:
app_files.append(os.path.join(parent, filename))
with open(checksum_file, 'a') as f:
for target_file in app_files:
f.write(self._gen_md5(target_file) + ' ' + target_file + '\n')
app_files.append('./' + checksum_file)
# gen application tarball
tarname = self._armada_app['appName'] + '-' + self._armada_app['version'] + '.tgz'
t = tarfile.open(tarname, 'w:gz')
for target_file in app_files:
t.add(target_file)
t.close()
os.chdir(store_cwd)
return tarname
# Generate armada application, including:
# 1. helm chart tarballs
# 2. armada manifest
# 3. metadata file
# 4. checksum file
# 5. application tarball
#
def gen_app(self, output_dir, overwrite):
ret = False
if not self._validate_app_attributes():
print('Error: Some of the app attributes are not valid!')
return ret
self._armada_app['outputDir'] = output_dir
self._armada_app['outputChartDir'] = output_dir + '/charts'
if not os.path.exists(self._armada_app['outputDir']):
os.makedirs(self._armada_app['outputDir'])
elif overwrite:
shutil.rmtree(self._armada_app['outputDir'])
else:
print('Output folder %s exists, please remove it or use --overwrite.' % self._armada_app['outputDir'])
return ret
if not os.path.exists(self._armada_app['outputChartDir']):
os.makedirs(self._armada_app['outputChartDir'])
# 1. Generating helm chart tarball
for chart in self._armada_chart:
ret = self._gen_helm_chart_tarball(chart)
if ret:
print('Helm chart %s tarball generated!' % chart['name'])
print('')
else:
print('Generating tarball for helm chart: %s error!' % chart['name'])
return ret
# 2. Generating armada manifest
ret = self._gen_armada_manifest()
if ret:
print('Armada manifest generated!')
else:
print('Armada manifest generation failed!')
return ret
# 3. Generating metadata file
ret = self._gen_metadata()
if ret:
print('Metadata generated!')
else:
print('Metadata generation failed!')
return ret
# 4&5. Generating checksum file and tarball
ret = self._gen_checksum_and_app_tarball()
if ret:
print('Checksum generated!')
print('App tarball generated at %s/%s' % (self._armada_app['outputDir'], ret))
print('')
else:
print('Checksum and App tarball generation failed!')
return ret
return ret
# For debug
def print_app_data(self):
print(self._armada_app)
print(self._armada_manifest)
print(self._armada_chartgroup)
print(self._armada_chart)
def parse_yaml(yaml_in):
yaml_data=''
try:
with open(yaml_in) as f:
yaml_data = yaml.safe_load(f)
except IOError:
print('Error: %s no found' % yaml_in )
except Exception as e:
print('Error: Invalid yaml file')
return yaml_data
def check_manifest(manifest_data):
# TODO: check more mandatory key/values in manifest yaml
# check app values
if 'appName' not in manifest_data:
print('Error: \'appName\' is missing.')
return False
if 'namespace' not in manifest_data:
print('Error: \'namespace\' is missing.')
return False
if 'version' not in manifest_data:
print('Error: \'version\' is missing.')
return False
# check manifest values
if 'manifest' not in manifest_data:
print('Error: \'manifest\'is missing.')
return False
if 'releasePrefix' not in manifest_data['manifest']:
print('Error: Manifest attribute \'releasePrefix\' is missing.')
return False
# check chartGroup values
if 'chartGroup' not in manifest_data:
print('Error: \'chartGroup\' is missing.')
return False
# check chart values
if 'chart' not in manifest_data:
print('Error: \'chart\' is missing.')
return False
for chart in manifest_data['chart']:
# check chart name
if 'name' not in chart:
print('Error: Chart attribute \'name\' is missing.')
return False
# check chart path, supporting: dir, git, tarball
if 'path' not in chart:
print('Error: Chart attribute \'path\' is missing in chart %s.' % chart['name'])
return False
else:
# TODO: To support branches/tags in git repo
if chart['path'].endswith('.git'):
if 'subpath' not in chart:
print('Error: Chart attribute \'subpath\' is missing in chart %s.' % chart['name'])
return False
chart['_pathType'] = 'git'
gitname = re.search('[^/]+(?=\.git$)',chart['path']).group()
if gitname:
chart['_gitname'] = gitname
else:
print('Error: Invalid \'path\' in chart %s.' % chart['name'])
print(' only \'local dir\', \'.git\', \'.tar.gz\', \'.tgz\' are supported')
return False
elif chart['path'].endswith('.tar.gz') or chart['path'].endswith('.tgz'):
if 'subpath' not in chart:
print('Error: Chart attribute \'subpath\' is missing in chart %s.' % chart['name'])
return False
chart['_pathType'] = 'tarball'
tarname = re.search('[^/]+(?=\.tgz)|[^/]+(?=\.tar\.gz)',chart['path']).group()
if tarname:
chart['_tarname'] = tarname
else:
print('Error: Invalid \'path\' in chart %s.' % chart['name'])
print(' only \'local dir\', \'.git\', \'.tar.gz\', \'.tgz\' are supported')
return False
else:
if not os.path.isdir(chart['path']):
print('Error: Invalid \'path\' in chart %s.' % chart['name'])
print(' only \'local dir\', \'.git\', \'.tar.gz\', \'.tgz\' are supported')
return False
chart['_pathType'] = 'dir'
return True
def generate_app(file_in, out_folder, overwrite):
global TEMP_APP_DIR
app_data = parse_yaml(file_in)
if not app_data:
print('Parse yaml error')
return
if not check_manifest(app_data):
print('Application manifest is not valid')
return
armada_app = ArmadaApplication(app_data)
TEMP_APP_DIR = TEMP_USER_DIR + armada_app.get_app_name() + '/'
app_out = out_folder + '/' + armada_app.get_app_name()
armada_app.gen_app(app_out, overwrite)
def main(argv):
input_file = ''
output_folder = '.'
overwrite = False
try:
options, args = getopt.getopt(argv, 'hi:o:', \
['help', 'input==', 'output==', 'overwrite'])
except getopt.GetoptError:
sys.exit()
for option, value in options:
if option in ('-h', '--help'):
print('StarlingX User Application Generator')
print('')
print('Usage:')
print(' python app-gen.py [Option]')
print('')
print('Options:')
print(' -i, --input yaml_file generate app from yaml_file')
print(' -o, --output folder generate app to output folder')
print(' --overwrite overwrite the output dir')
print(' -h, --help this help')
if option in ('--overwrite'):
overwrite = True
if option in ('-i', '--input'):
input_file = value
if option in ('-o', '--output'):
output_folder = value
if not os.path.isfile(os.path.abspath(input_file)):
print('Error: input file not found')
sys.exit()
if input_file:
generate_app(os.path.abspath(input_file), os.path.abspath(output_folder), overwrite)
if __name__ == '__main__':
main(sys.argv[1:])

View File

@ -0,0 +1,39 @@
#!/bin/bash
OP=$1
CHARTDIR=$2
if [ -z "$OP"]; then
exit 1
fi
if [ -z "$CHARTDIR"]; then
exit 1
fi
if [ $OP == "waitlabel" ]; then
KIND=("Deployment" "StatefulSet" "DaemonSet")
if [ ! -d $CHARTDIR/templates ]; then
exit 1
fi
cd $CHARTDIR/templates
for target in ${KIND[@]}; do
output=$(grep $target . -rn | awk -F ':' '{print$1}' \
| xargs awk -F ':' '/{{ .Release.Name }}/{print$1; exit}')
if [ "x$output" != "x" ]; then
echo $output
exit 0
fi
done
elif [ $OP == "chartname" ]; then
if [ ! -f $CHARTDIR/Chart.yaml ]; then
exit 1
fi
cd $CHARTDIR
output=$(awk '/name:/{print$2;exit}' Chart.yaml)
if [ "x$output" != "x" ]; then
echo $output
exit 0
fi
fi
exit 1

32
app-gen-tool/example.yaml Normal file
View File

@ -0,0 +1,32 @@
---
appName: stx-app
namespace: stx-app
version: 1.0-1
chart:
- name: chart1
path: /path/to/chart1
wait: 600
values:
test_key: test_value
- name: chart2
path: https://git/of/chart2.git
- name: chart3
path: https://tarball/of/chart3-sha.tgz
chartGroup:
- name: chartgroup1
description: "This is the first chartgroup"
sequenced: true
chart_group:
- chart1
- chart2
- name: chartgroup2
description: "This is the second chartgroup"
sequenced: false
chart_group:
- chart3
manifest:
name: stx-app-manifest
releasePrefix: myprefix
chart_groups:
- chartgroup1
- chartgroup2

View File

@ -0,0 +1,26 @@
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: $NAME$
data:
chart_name: $NAME$
release: $NAME$
namespace: $NAMESPACE$
test:
enabled: false
wait:
timeout: $WAIT%600$
labels:
$WAIT_LABEL_KEY$: $RELEASE_PREFIX$-$NAME$
install:
no_hooks: false
upgrade:
no_hooks: false
@VALUES|2@
source:
type: tar
location: http://172.17.0.1:8080/helm_charts/starlingx/$TARBALL_NAME$
subpath: $CHART_ARCNAME$
dependencies: []

View File

@ -0,0 +1,10 @@
---
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: $NAME$
data:
description: $DESCRIPTION%This is a chartgroup$
sequenced: $SEQUENCED%false$
@CHART_GROUP|2@

View File

@ -0,0 +1,8 @@
---
schema: armada/Manifest/v1
metadata:
schema: metadata/Document/v1
name: $NAME$
data:
release_prefix: $RELEASE_PREFIX$
@CHART_GROUPS|2@