From 3c674e0cc70efcde17bc2eeb7a4154a191172246 Mon Sep 17 00:00:00 2001 From: Ovidiu Poncea Date: Thu, 27 Dec 2018 13:53:34 +0200 Subject: [PATCH] Replace compiled crushmaps with human readable versions This will simplify changes to the crushmap as we can keep them as text in git. Change-Id: I8957f27a7880fe1a7ee2f8102eb507fb9e950a6e Story: 2002844 Task: 28723 Depends-On: https://review.openstack.org/629511 Signed-off-by: Ovidiu Poncea --- sysinv/sysinv/centos/sysinv.spec | 9 ++- .../sysinv/etc/sysinv/crushmap-aio-dx.bin | Bin 485 -> 0 bytes .../sysinv/etc/sysinv/crushmap-aio-sx.bin | Bin 437 -> 0 bytes .../sysinv/etc/sysinv/crushmap-aio-sx.txt | 58 ++++++++++++++++ .../etc/sysinv/crushmap-controller-model.txt | 65 ++++++++++++++++++ .../etc/sysinv/crushmap-storage-model.txt | 65 ++++++++++++++++++ sysinv/sysinv/sysinv/etc/sysinv/crushmap.bin | Bin 479 -> 0 bytes .../sysinv/api/controllers/v1/storage_tier.py | 2 +- sysinv/sysinv/sysinv/sysinv/common/ceph.py | 19 +++-- 9 files changed, 206 insertions(+), 12 deletions(-) delete mode 100644 sysinv/sysinv/sysinv/etc/sysinv/crushmap-aio-dx.bin delete mode 100644 sysinv/sysinv/sysinv/etc/sysinv/crushmap-aio-sx.bin create mode 100644 sysinv/sysinv/sysinv/etc/sysinv/crushmap-aio-sx.txt create mode 100644 sysinv/sysinv/sysinv/etc/sysinv/crushmap-controller-model.txt create mode 100644 sysinv/sysinv/sysinv/etc/sysinv/crushmap-storage-model.txt delete mode 100644 sysinv/sysinv/sysinv/etc/sysinv/crushmap.bin diff --git a/sysinv/sysinv/centos/sysinv.spec b/sysinv/sysinv/centos/sysinv.spec index 24f054615a..3c9e8924ce 100644 --- a/sysinv/sysinv/centos/sysinv.spec +++ b/sysinv/sysinv/centos/sysinv.spec @@ -61,11 +61,10 @@ install -p -D -m 755 etc/sysinv/sysinv_goenabled_check.sh %{buildroot}%{local_et install -d -m 755 %{buildroot}%{local_etc_sysinv} install -p -D -m 755 etc/sysinv/policy.json %{buildroot}%{local_etc_sysinv}/policy.json install -p -D -m 640 etc/sysinv/profileSchema.xsd %{buildroot}%{local_etc_sysinv}/profileSchema.xsd -#In order to decompile crushmap.bin please use this command: -#crushtool -d crushmap.bin -o {decompiled-crushmap-filename} -install -p -D -m 655 etc/sysinv/crushmap.bin %{buildroot}%{local_etc_sysinv}/crushmap.bin -install -p -D -m 655 etc/sysinv/crushmap-aio-sx.bin %{buildroot}%{local_etc_sysinv}/crushmap-aio-sx.bin -install -p -D -m 655 etc/sysinv/crushmap-aio-dx.bin %{buildroot}%{local_etc_sysinv}/crushmap-aio-dx.bin + +install -p -D -m 644 etc/sysinv/crushmap-storage-model.txt %{buildroot}%{local_etc_sysinv}/crushmap-storage-model.txt +install -p -D -m 644 etc/sysinv/crushmap-controller-model.txt %{buildroot}%{local_etc_sysinv}/crushmap-controller-model.txt +install -p -D -m 644 etc/sysinv/crushmap-aio-sx.txt %{buildroot}%{local_etc_sysinv}/crushmap-aio-sx.txt install -d -m 755 %{buildroot}%{local_etc_motdd} install -p -D -m 755 etc/sysinv/motd-system %{buildroot}%{local_etc_motdd}/10-system diff --git a/sysinv/sysinv/sysinv/etc/sysinv/crushmap-aio-dx.bin b/sysinv/sysinv/sysinv/etc/sysinv/crushmap-aio-dx.bin deleted file mode 100644 index 6069205a7e7fde9d5727d7ba9ab3f7b52692d84e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 485 zcmZ8d?P|j?479s;UDN#>dy7E37b+$(X=oM&*`Y7Le%v`u2@^=?JDrg&W32fw26c)x z;DIO%Pm_KJPXvW3`6~BU;D6w^g=gudZnKv<@6*53X8(+QOPX$N!87b>K4*~=-;s=$ zccNyW@eL@Up4(>$k=M4!>g+;j109^AuDTG(Rj!+S9O{uXs18!|UNVdPa$NDkNPNaS}vftB=8rR%(w2d?FxcH~+)r%@S)@&J}BA&R=jm$CKA?=ZfQ WSwB%f>Dc+;V``%OAnCRCo6-|02Vcbi diff --git a/sysinv/sysinv/sysinv/etc/sysinv/crushmap-aio-sx.bin b/sysinv/sysinv/sysinv/etc/sysinv/crushmap-aio-sx.bin deleted file mode 100644 index 79177b162290e0297c7442aeb0f51133c49d6c1f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 437 zcmZ8d?Fxf13~b%E+t0DL5M(bTMT^Ruk+ia>-$$;kFbpJ@yCh9prL;OKB{AAo;479Z zWKsVJ4@3e~@a(VtUe&>MsxNbE#J;dW>m0d^gdLgW;K053V(~VSos+l`B-boQb>uw8 z4&rrZoa-GrFv+HV0Ku7FyOd7Nj2%!XYA_d?iJth9T)Z*f)OPfiPr}g5w(V&|E%kbg z)$--I(;rg-~9iW2VP=+*iL F=?6ZmPapsQ diff --git a/sysinv/sysinv/sysinv/etc/sysinv/crushmap-aio-sx.txt b/sysinv/sysinv/sysinv/etc/sysinv/crushmap-aio-sx.txt new file mode 100644 index 0000000000..fdf25c5cf4 --- /dev/null +++ b/sysinv/sysinv/sysinv/etc/sysinv/crushmap-aio-sx.txt @@ -0,0 +1,58 @@ +# begin crush map +tunable choose_local_tries 0 +tunable choose_local_fallback_tries 0 +tunable choose_total_tries 50 +tunable chooseleaf_descend_once 1 +tunable chooseleaf_vary_r 1 +tunable straw_calc_version 1 + +# devices + +# types +type 0 osd +type 1 host +type 2 chassis +type 3 rack +type 4 row +type 5 pdu +type 6 pod +type 7 room +type 8 datacenter +type 9 region +type 10 root + +# buckets +host controller-0 { + id -3 # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 +} +chassis group-0 { + id -2 # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 + item controller-0 weight 0.000 +} +root storage-tier { + id -1 # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 + item group-0 weight 0.000 +} + +# rules +rule storage_tier_ruleset { + ruleset 0 + type replicated + min_size 1 + max_size 10 + step take storage-tier + step choose firstn 1 type chassis + step chooseleaf firstn 0 type osd + step emit +} + +# end crush map diff --git a/sysinv/sysinv/sysinv/etc/sysinv/crushmap-controller-model.txt b/sysinv/sysinv/sysinv/etc/sysinv/crushmap-controller-model.txt new file mode 100644 index 0000000000..94add7c08c --- /dev/null +++ b/sysinv/sysinv/sysinv/etc/sysinv/crushmap-controller-model.txt @@ -0,0 +1,65 @@ +# begin crush map +tunable choose_local_tries 0 +tunable choose_local_fallback_tries 0 +tunable choose_total_tries 50 +tunable chooseleaf_descend_once 1 +tunable chooseleaf_vary_r 1 +tunable straw_calc_version 1 + +# devices + +# types +type 0 osd +type 1 host +type 2 chassis +type 3 rack +type 4 row +type 5 pdu +type 6 pod +type 7 room +type 8 datacenter +type 9 region +type 10 root + +# buckets +host controller-0 { + id -4 # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 +} +host controller-1 { + id -3 # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 +} +chassis group-0 { + id -2 # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 + item controller-0 weight 0.000 + item controller-1 weight 0.000 +} +root storage-tier { + id -1 # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 + item group-0 weight 0.000 +} + +# rules +rule storage_tier_ruleset { + ruleset 0 + type replicated + min_size 1 + max_size 10 + step take storage-tier + step choose firstn 1 type chassis + step chooseleaf firstn 0 type host + step emit +} + +# end crush map diff --git a/sysinv/sysinv/sysinv/etc/sysinv/crushmap-storage-model.txt b/sysinv/sysinv/sysinv/etc/sysinv/crushmap-storage-model.txt new file mode 100644 index 0000000000..620c676f79 --- /dev/null +++ b/sysinv/sysinv/sysinv/etc/sysinv/crushmap-storage-model.txt @@ -0,0 +1,65 @@ +# begin crush map +tunable choose_local_tries 0 +tunable choose_local_fallback_tries 0 +tunable choose_total_tries 50 +tunable chooseleaf_descend_once 1 +tunable chooseleaf_vary_r 1 +tunable straw_calc_version 1 + +# devices + +# types +type 0 osd +type 1 host +type 2 chassis +type 3 rack +type 4 row +type 5 pdu +type 6 pod +type 7 room +type 8 datacenter +type 9 region +type 10 root + +# buckets +host storage-0 { + id -4 # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 +} +host storage-1 { + id -5 # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 +} +chassis group-0 { + id -3 # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 + item storage-0 weight 0.000 + item storage-1 weight 0.000 +} +root storage-tier { + id -1 # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 + item group-0 weight 0.000 +} + +# rules +rule storage_tier_ruleset { + ruleset 0 + type replicated + min_size 1 + max_size 10 + step take storage-tier + step choose firstn 1 type chassis + step chooseleaf firstn 0 type host + step emit +} + +# end crush map diff --git a/sysinv/sysinv/sysinv/etc/sysinv/crushmap.bin b/sysinv/sysinv/sysinv/etc/sysinv/crushmap.bin deleted file mode 100644 index abddc8fea43bb0d00e3306dfaf6b344d76d9785c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 479 zcmZ9I(Qd*p3`G+dgLOPk`wJw-522`yN(IAJl4>6hi20=HRLzp(-0PFNi80n}jL~(B z7cEbX%CH;#hw3cKLI-bAn1$-Q>f1jK9G0Ff?Bv1DJ^U&7mpt9tioK$FK6B92S54xN z9i@3Mb*WWiGqq0=6UWw?^~uH9MF^$@SD&fqAU~_J?VEuaxVKtY0>LT^IlW diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/storage_tier.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/storage_tier.py index 4acb62e4ab..4eaf4ed542 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/storage_tier.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/storage_tier.py @@ -504,7 +504,7 @@ def _create(self, tier, iprofile=None): try: self._ceph.set_crushmap() except exception.CephCrushMapNotApplied as e: - LOG.warning("Crushmap not applied, seems like ceph cluster is not ""configured. " + LOG.warning("Crushmap not applied, seems like ceph cluster is not configured. " "Operation will be retried with first occasion. " "Reason: %s" % str(e)) else: diff --git a/sysinv/sysinv/sysinv/sysinv/common/ceph.py b/sysinv/sysinv/sysinv/sysinv/common/ceph.py index 03f843b9b5..42641de1e6 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/ceph.py +++ b/sysinv/sysinv/sysinv/sysinv/common/ceph.py @@ -557,7 +557,7 @@ class CephApiOperator(object): hostupdate.ihost_orig['invprovision'] != constants.PROVISIONED): - # update crushmap.bin accordingly with the host and it's peer group + # update crushmap accordingly with the host and it's peer group node_bucket = hostupdate.ihost_orig['hostname'] ipeer = pecan.request.dbapi.peer_get( hostupdate.ihost_orig['peer_id']) @@ -709,14 +709,21 @@ def fix_crushmap(dbapi=None): constants.CEPH_CRUSH_MAP_APPLIED) if not os.path.isfile(crushmap_flag_file): if utils.is_aio_simplex_system(dbapi): - crushmap_file = "/etc/sysinv/crushmap-aio-sx.bin" + crushmap_txt = "/etc/sysinv/crushmap-aio-sx.txt" elif utils.is_aio_duplex_system(dbapi): - crushmap_file = "/etc/sysinv/crushmap-aio-dx.bin" + crushmap_txt = "/etc/sysinv/crushmap-controller-model.txt" else: - crushmap_file = "/etc/sysinv/crushmap.bin" - LOG.info("Updating crushmap with: %s" % crushmap_file) + crushmap_txt = "/etc/sysinv/crushmap-storage-model.txt" + LOG.info("Updating crushmap with: %s" % crushmap_txt) + try: - subprocess.check_output("ceph osd setcrushmap -i %s" % crushmap_file, + # Compile crushmap + crushmap_bin = "/etc/sysinv/crushmap.bin" + subprocess.check_output("crushtool -c %s " + "-o %s" % (crushmap_txt, crushmap_bin), + stderr=subprocess.STDOUT, shell=True) + # Set crushmap + subprocess.check_output("ceph osd setcrushmap -i %s" % crushmap_bin, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: # May not be critical, depends on where this is called.