diff --git a/sysinv/sysinv/centos/sysinv.spec b/sysinv/sysinv/centos/sysinv.spec index 24f054615a..3c9e8924ce 100644 --- a/sysinv/sysinv/centos/sysinv.spec +++ b/sysinv/sysinv/centos/sysinv.spec @@ -61,11 +61,10 @@ install -p -D -m 755 etc/sysinv/sysinv_goenabled_check.sh %{buildroot}%{local_et install -d -m 755 %{buildroot}%{local_etc_sysinv} install -p -D -m 755 etc/sysinv/policy.json %{buildroot}%{local_etc_sysinv}/policy.json install -p -D -m 640 etc/sysinv/profileSchema.xsd %{buildroot}%{local_etc_sysinv}/profileSchema.xsd -#In order to decompile crushmap.bin please use this command: -#crushtool -d crushmap.bin -o {decompiled-crushmap-filename} -install -p -D -m 655 etc/sysinv/crushmap.bin %{buildroot}%{local_etc_sysinv}/crushmap.bin -install -p -D -m 655 etc/sysinv/crushmap-aio-sx.bin %{buildroot}%{local_etc_sysinv}/crushmap-aio-sx.bin -install -p -D -m 655 etc/sysinv/crushmap-aio-dx.bin %{buildroot}%{local_etc_sysinv}/crushmap-aio-dx.bin + +install -p -D -m 644 etc/sysinv/crushmap-storage-model.txt %{buildroot}%{local_etc_sysinv}/crushmap-storage-model.txt +install -p -D -m 644 etc/sysinv/crushmap-controller-model.txt %{buildroot}%{local_etc_sysinv}/crushmap-controller-model.txt +install -p -D -m 644 etc/sysinv/crushmap-aio-sx.txt %{buildroot}%{local_etc_sysinv}/crushmap-aio-sx.txt install -d -m 755 %{buildroot}%{local_etc_motdd} install -p -D -m 755 etc/sysinv/motd-system %{buildroot}%{local_etc_motdd}/10-system diff --git a/sysinv/sysinv/sysinv/etc/sysinv/crushmap-aio-dx.bin b/sysinv/sysinv/sysinv/etc/sysinv/crushmap-aio-dx.bin deleted file mode 100644 index 6069205a7e..0000000000 Binary files a/sysinv/sysinv/sysinv/etc/sysinv/crushmap-aio-dx.bin and /dev/null differ diff --git a/sysinv/sysinv/sysinv/etc/sysinv/crushmap-aio-sx.bin b/sysinv/sysinv/sysinv/etc/sysinv/crushmap-aio-sx.bin deleted file mode 100644 index 79177b1622..0000000000 Binary files a/sysinv/sysinv/sysinv/etc/sysinv/crushmap-aio-sx.bin and /dev/null differ diff --git a/sysinv/sysinv/sysinv/etc/sysinv/crushmap-aio-sx.txt b/sysinv/sysinv/sysinv/etc/sysinv/crushmap-aio-sx.txt new file mode 100644 index 0000000000..fdf25c5cf4 --- /dev/null +++ b/sysinv/sysinv/sysinv/etc/sysinv/crushmap-aio-sx.txt @@ -0,0 +1,58 @@ +# begin crush map +tunable choose_local_tries 0 +tunable choose_local_fallback_tries 0 +tunable choose_total_tries 50 +tunable chooseleaf_descend_once 1 +tunable chooseleaf_vary_r 1 +tunable straw_calc_version 1 + +# devices + +# types +type 0 osd +type 1 host +type 2 chassis +type 3 rack +type 4 row +type 5 pdu +type 6 pod +type 7 room +type 8 datacenter +type 9 region +type 10 root + +# buckets +host controller-0 { + id -3 # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 +} +chassis group-0 { + id -2 # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 + item controller-0 weight 0.000 +} +root storage-tier { + id -1 # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 + item group-0 weight 0.000 +} + +# rules +rule storage_tier_ruleset { + ruleset 0 + type replicated + min_size 1 + max_size 10 + step take storage-tier + step choose firstn 1 type chassis + step chooseleaf firstn 0 type osd + step emit +} + +# end crush map diff --git a/sysinv/sysinv/sysinv/etc/sysinv/crushmap-controller-model.txt b/sysinv/sysinv/sysinv/etc/sysinv/crushmap-controller-model.txt new file mode 100644 index 0000000000..94add7c08c --- /dev/null +++ b/sysinv/sysinv/sysinv/etc/sysinv/crushmap-controller-model.txt @@ -0,0 +1,65 @@ +# begin crush map +tunable choose_local_tries 0 +tunable choose_local_fallback_tries 0 +tunable choose_total_tries 50 +tunable chooseleaf_descend_once 1 +tunable chooseleaf_vary_r 1 +tunable straw_calc_version 1 + +# devices + +# types +type 0 osd +type 1 host +type 2 chassis +type 3 rack +type 4 row +type 5 pdu +type 6 pod +type 7 room +type 8 datacenter +type 9 region +type 10 root + +# buckets +host controller-0 { + id -4 # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 +} +host controller-1 { + id -3 # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 +} +chassis group-0 { + id -2 # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 + item controller-0 weight 0.000 + item controller-1 weight 0.000 +} +root storage-tier { + id -1 # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 + item group-0 weight 0.000 +} + +# rules +rule storage_tier_ruleset { + ruleset 0 + type replicated + min_size 1 + max_size 10 + step take storage-tier + step choose firstn 1 type chassis + step chooseleaf firstn 0 type host + step emit +} + +# end crush map diff --git a/sysinv/sysinv/sysinv/etc/sysinv/crushmap-storage-model.txt b/sysinv/sysinv/sysinv/etc/sysinv/crushmap-storage-model.txt new file mode 100644 index 0000000000..620c676f79 --- /dev/null +++ b/sysinv/sysinv/sysinv/etc/sysinv/crushmap-storage-model.txt @@ -0,0 +1,65 @@ +# begin crush map +tunable choose_local_tries 0 +tunable choose_local_fallback_tries 0 +tunable choose_total_tries 50 +tunable chooseleaf_descend_once 1 +tunable chooseleaf_vary_r 1 +tunable straw_calc_version 1 + +# devices + +# types +type 0 osd +type 1 host +type 2 chassis +type 3 rack +type 4 row +type 5 pdu +type 6 pod +type 7 room +type 8 datacenter +type 9 region +type 10 root + +# buckets +host storage-0 { + id -4 # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 +} +host storage-1 { + id -5 # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 +} +chassis group-0 { + id -3 # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 + item storage-0 weight 0.000 + item storage-1 weight 0.000 +} +root storage-tier { + id -1 # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 + item group-0 weight 0.000 +} + +# rules +rule storage_tier_ruleset { + ruleset 0 + type replicated + min_size 1 + max_size 10 + step take storage-tier + step choose firstn 1 type chassis + step chooseleaf firstn 0 type host + step emit +} + +# end crush map diff --git a/sysinv/sysinv/sysinv/etc/sysinv/crushmap.bin b/sysinv/sysinv/sysinv/etc/sysinv/crushmap.bin deleted file mode 100644 index abddc8fea4..0000000000 Binary files a/sysinv/sysinv/sysinv/etc/sysinv/crushmap.bin and /dev/null differ diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/storage_tier.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/storage_tier.py index 4acb62e4ab..4eaf4ed542 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/storage_tier.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/storage_tier.py @@ -504,7 +504,7 @@ def _create(self, tier, iprofile=None): try: self._ceph.set_crushmap() except exception.CephCrushMapNotApplied as e: - LOG.warning("Crushmap not applied, seems like ceph cluster is not ""configured. " + LOG.warning("Crushmap not applied, seems like ceph cluster is not configured. " "Operation will be retried with first occasion. " "Reason: %s" % str(e)) else: diff --git a/sysinv/sysinv/sysinv/sysinv/common/ceph.py b/sysinv/sysinv/sysinv/sysinv/common/ceph.py index 03f843b9b5..42641de1e6 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/ceph.py +++ b/sysinv/sysinv/sysinv/sysinv/common/ceph.py @@ -557,7 +557,7 @@ class CephApiOperator(object): hostupdate.ihost_orig['invprovision'] != constants.PROVISIONED): - # update crushmap.bin accordingly with the host and it's peer group + # update crushmap accordingly with the host and it's peer group node_bucket = hostupdate.ihost_orig['hostname'] ipeer = pecan.request.dbapi.peer_get( hostupdate.ihost_orig['peer_id']) @@ -709,14 +709,21 @@ def fix_crushmap(dbapi=None): constants.CEPH_CRUSH_MAP_APPLIED) if not os.path.isfile(crushmap_flag_file): if utils.is_aio_simplex_system(dbapi): - crushmap_file = "/etc/sysinv/crushmap-aio-sx.bin" + crushmap_txt = "/etc/sysinv/crushmap-aio-sx.txt" elif utils.is_aio_duplex_system(dbapi): - crushmap_file = "/etc/sysinv/crushmap-aio-dx.bin" + crushmap_txt = "/etc/sysinv/crushmap-controller-model.txt" else: - crushmap_file = "/etc/sysinv/crushmap.bin" - LOG.info("Updating crushmap with: %s" % crushmap_file) + crushmap_txt = "/etc/sysinv/crushmap-storage-model.txt" + LOG.info("Updating crushmap with: %s" % crushmap_txt) + try: - subprocess.check_output("ceph osd setcrushmap -i %s" % crushmap_file, + # Compile crushmap + crushmap_bin = "/etc/sysinv/crushmap.bin" + subprocess.check_output("crushtool -c %s " + "-o %s" % (crushmap_txt, crushmap_bin), + stderr=subprocess.STDOUT, shell=True) + # Set crushmap + subprocess.check_output("ceph osd setcrushmap -i %s" % crushmap_bin, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: # May not be critical, depends on where this is called.