diff --git a/docs/worker_ansible_howto.md b/docs/worker_ansible_howto.md index 2fa5473e..80bebb6f 100755 --- a/docs/worker_ansible_howto.md +++ b/docs/worker_ansible_howto.md @@ -552,6 +552,31 @@ Ansible controller node. Group `etcdnodes` should list all the servers that you etcd services on. Group `builders` should contain just one server that you wish to use to build Hyperledger Fabric artifacts such as executables and docker images. +## Using existing k8s cluster + +Ansible agent allows you to deploy fabric network onto an existing k8s cluster. To do that, +you only need to place your k8s configuration file and possibly certificates in the ansible +agent vars directory, the kube configuration file must be named kubeconfig. Then you can +use exactly same fabric network configuration file such as bc1st.yml, vb1st.yml file to +stand up your own fabric network. You can use the sample configuration file like bc1st.yml or +you can use these sample configuration as a starting point to create new ones. Then you simply +run the following command to stand up your own fabric network. + +``` + ansible-playbook -e "mode=apply env=bc1st deploy_type=k8s" setupfabric.yml +``` + +The above command will use the vars/kubeconfig file to deploy fabric network defined in +vars/bc1st.yml file. + +To destroy what you just created, run the following command:: + +``` + ansible-playbook -e "mode=destroy env=bc1st deploy_type=k8s" setupfabric.yml +``` + +You can also choose to use the ansible agent container to run the above command. + ## Required Ports And Security Considerations When you work with the public cloud, it is important to open or close certain diff --git a/src/agent/ansible/roles/deploy_compose/certsetup/templates/configtx.j2 b/src/agent/ansible/roles/deploy_compose/certsetup/templates/configtx.j2 index fe748d4a..6d4d8ca8 100755 --- a/src/agent/ansible/roles/deploy_compose/certsetup/templates/configtx.j2 +++ b/src/agent/ansible/roles/deploy_compose/certsetup/templates/configtx.j2 @@ -1,4 +1,5 @@ --- +{% set project_version = fabric.baseimage_tag %} {% if allorgs | length > 0 %} Organizations: {% for org in allorgs %} diff --git a/src/agent/ansible/roles/deploy_compose/certsetup/templates/core.j2 b/src/agent/ansible/roles/deploy_compose/certsetup/templates/core.j2 index 30d64499..f37be92e 100755 --- a/src/agent/ansible/roles/deploy_compose/certsetup/templates/core.j2 +++ b/src/agent/ansible/roles/deploy_compose/certsetup/templates/core.j2 @@ -1,5 +1,6 @@ --- logging: +{% set project_version = fabric.baseimage_tag %} {% if project_version | version_compare('1.1.0','>=') %} level: info {% else %} diff --git a/src/agent/ansible/roles/deploy_k8s/fabricsetup/tasks/apply.yml b/src/agent/ansible/roles/deploy_k8s/fabricsetup/tasks/apply.yml index a2b406be..e3da474c 100755 --- a/src/agent/ansible/roles/deploy_k8s/fabricsetup/tasks/apply.yml +++ b/src/agent/ansible/roles/deploy_k8s/fabricsetup/tasks/apply.yml @@ -1,8 +1,8 @@ --- - name: Setup and initialize variables set_fact: - current_host: "{{ hostvars[inventory_hostname].inter_name }}" fabricworkdir: "/opt/gopath/{{ env }}/fabric" + fabricpath: "/opt/gopath/{{ env }}/src/github.com/hyperledger/fabric" gopath: "/opt/gopath/{{ env }}" peers: "{{ [] }}" orderers: "{{ [] }}" @@ -23,7 +23,6 @@ allpeers: "{{ [] }}" clihost: "" filterstr: "" - k8s_server: "{{ hostvars[groups['allnodes'][0]].private_ip }}" - name: Make sure that working directory exists and clean file: @@ -88,171 +87,127 @@ 'name':item | replace('.', '-') }] }} with_items: "{{ ordererls }}" -- name: Get peer container list - set_fact: - peers: | - {{ peers + [{'org':item.split('@')[1].split('.')[-1], - 'name':item.split('@')[1] | replace('.', '-'), - 'role':item.split('@')[0]}] }} - with_items: "{{ fabric.network[current_host].peers | default([]) }}" - -- name: Get ca container list - set_fact: - cas: | - {{ cas + [{'org':item.split('.')[-1], - 'name':item | replace('.', '-') }] }} - with_items: "{{ fabric.network[current_host].cas | default([]) }}" +- name: Query k8s storage class + command: >- + ./kubectl --kubeconfig kubeconfig get storageclass + args: + chdir: "{{ playbook_dir }}/../../vars/" + register: classes -- name: Get orderer container list +- name: Set storage class set_fact: - orderers: | - {{ orderers + [{'org':item.split('.')[-1], - 'name':item | replace('.', '-') }] }} - with_items: "{{ fabric.network[current_host].orderers | default([]) }}" + storageclass: "{{ (classes.stderr.find('No resources found') >= 0) | ternary('', 'default') }}" -- name: Set zookeeper and kafka container list - set_fact: - zookeepers: "{{ fabric.network[current_host].zookeepers | default([]) }}" - kafkas: "{{ fabric.network[current_host].kafkas | default([]) }}" +- name: Create deployment files for certssetup and pvc + template: + src: "{{ playbook_dir }}/../deploy_k8s/fabricsetup/templates/{{ item }}.j2" + dest: "{{ playbook_dir }}/../../run/{{ item }}.yaml" + with_items: + - pvc + - certssetup -- name: Pull certificates from the builder machine +- name: Create persistent volume command: >- - scp -i "/opt/gopath/id_rsa" -r -o "StrictHostKeyChecking no" "{{ fabric. - ssh_user }}@{{ hostvars[groups['builders'][0]].private_ip }}:{{ fabricworkdir }}/certs.tgz" - "{{ fabricworkdir }}/allcerts.tgz" + ./kubectl --kubeconfig kubeconfig apply -f ../run/pvc.yaml + args: + chdir: "{{ playbook_dir }}/../../vars/" + tags: "createpvc" -- name: Pull container images from the builder machine +- name: Query PVC status command: >- - scp -i "/opt/gopath/id_rsa" -r -o "StrictHostKeyChecking no" "{{ fabric. - ssh_user }}@{{ hostvars[groups['builders'][0]].private_ip }}:{{ fabricworkdir }}/images/fabricimages.tar" - "{{ fabricworkdir }}/fabricimages.tar" - when: fabric.baseimage_tag == '' and inventory_hostname not in groups['builders'] - -- name: Pull container version file from the build machine + ./kubectl --kubeconfig kubeconfig get -o=custom-columns=STATUS:.status.phase + pvc fabriccerts + args: + chdir: "{{ playbook_dir }}/../../vars/" + register: pvcstatus + until: pvcstatus.stdout.find("Bound") >= 0 + retries: 5 + delay: 10 + tags: "querypvc" + +- name: Create certs volume initialization pod command: >- - scp -i "/opt/gopath/id_rsa" -r -o "StrictHostKeyChecking no" "{{ fabric. - ssh_user }}@{{ hostvars[groups['builders'][0]].private_ip }}:{{ fabricworkdir }}/images/VERSION" - "{{ fabricworkdir }}/VERSION" - -- stat: - path: "{{ fabricworkdir }}/fabricimages.tar" - register: imagepack - -- name: Load all the docker images created by build machine - shell: >- - docker load -i {{ fabricworkdir }}/fabricimages.tar - when: imagepack.stat.exists == true and inventory_hostname not in groups['builders'] + ./kubectl --kubeconfig kubeconfig apply -f ../run/certssetup.yaml + args: + chdir: "{{ playbook_dir }}/../../vars/" + tags: "createcertspod" -- name: Find out the image tags - slurp: - src: "{{ fabricworkdir }}/VERSION" - register: imagetag - -- name: Set image tag from the file - set_fact: - thetag: "{{ imagetag['content'] | b64decode }}" - helpertag: "{{ fabric.helper_tag }}" - -- name: Unpack the certificates - unarchive: - src: "{{ fabricworkdir }}/allcerts.tgz" - dest: "{{ fabricworkdir }}/run" - remote_src: true - -- name: Process private key files - template: - src: "{{ playbook_dir }}/../deploy_k8s/fabricsetup/templates/fabric-ca-server-config.j2" - dest: "{{ fabricworkdir }}/run/keyfiles/{{ item.org }}/ca/fabric-ca-server-config.yaml" - with_items: "{{ cas }}" - -- name: Get the peer org list +- name: Query initialization container status + command: >- + ./kubectl --kubeconfig kubeconfig get -o=custom-columns=STATUS:.status.phase + pod fabriccertspod + args: + chdir: "{{ playbook_dir }}/../../vars/" + register: pvcstatus + until: pvcstatus.stdout.find("Running") >= 0 + retries: 5 + delay: 10 + tags: "querypvc" + +- name: Set peer and orderer to run peer channel create command set_fact: - peerorgs: "{{ peers | map(attribute='org') | list | unique | sort }}" + clipeer: "{{ allpeers | random }}" + cliorderer: "{{ allorderers | random }}" + when: peerls | length > 0 - name: Get all peer orgs set_fact: - orgmembers: "{{ peers | map(attribute='org') | list | unique | sort | join(\".member' '\") | trim | replace(' ', ',') }}" + orgmembers: "{{ allpeers | map(attribute='org') | list | unique | sort | join(\".member' '\") | trim | replace(' ', ',') }}" -- name: Create k8s deployment files +- name: Create peer channel command script template: - src: "{{ playbook_dir }}/../deploy_k8s/fabricsetup/templates/fabric-pod.j2" - dest: "{{ fabricworkdir }}/run/fabric-pod.yml" + src: "{{ playbook_dir }}/../deploy_k8s/fabricsetup/templates/dochannel.j2" + dest: "{{ fabricworkdir }}/keyfiles/dochannel.sh" + mode: "u=rx,g=rx" -- name: Get pod filter string from peers and orderes - set_fact: - filterstr : "{{ filterstr + ' -e k8s_' + item.name }}" +- name: Copy certs onto the persistent volume + command: >- + ./kubectl --kubeconfig kubeconfig cp {{ item }} fabriccertspod:/fabriccerts + args: + chdir: "{{ playbook_dir }}/../../vars/" with_items: - - "{{ peers }}" - - "{{ orderers }}" + - "{{ fabricworkdir }}/certs.tgz" + - "{{ fabricworkdir }}/keyfiles/dochannel.sh" + - "{{ playbook_dir }}/../deploy_k8s/fabricsetup/templates/firstcode.go" + tags: "placecerts" -- name: Get pod filter string from zookeepers and kafkas - set_fact: - filterstr : "{{ filterstr + ' -e k8s_' + item }}" - with_items: - - "{{ zookeepers }}" - - "{{ kafkas }}" - - "fabriccli" +- name: Untar certs.tgz in the Persistent volume + command: >- + ./kubectl --kubeconfig kubeconfig exec fabriccertspod -c task-pv-container + -- tar -C /fabriccerts -xzvf /fabriccerts/certs.tgz + args: + chdir: "{{ playbook_dir }}/../../vars/" + tags: "untarcerts" -- name: Pull necessary container images from the docker hub - command: "docker pull {{ fabric.repo.url }}{{ item.name }}" - when: item.flag | length > 0 and fabric.baseimage_tag | length > 0 - with_items: - - { name: "fabric-ca:{{ fabric.ca.image_tag | default(thetag) }}", flag: "{{ cas }}" } - - { name: "fabric-zookeeper:{{ helpertag }}", flag: "{{ zookeepers }}" } - - { name: "fabric-kafka:{{ helpertag }}", flag: "{{ kafkas }}" } - - { name: "fabric-couchdb:{{ helpertag }}", flag: "{{ peers }}" } - - { name: "fabric-orderer:{{ thetag }}", flag: "{{ orderers }}" } - - { name: "fabric-peer:{{ thetag }}", flag: "{{ peers }}" } - - { name: "fabric-ccenv:{{ thetag }}", flag: "{{ peers }}" } - - { name: "fabric-tools:{{ thetag }}", flag: "tools" } - tags: "pullimages" +- name: Removing the certs pod + command: >- + ./kubectl --kubeconfig kubeconfig delete -f ../run/certssetup.yaml + args: + chdir: "{{ playbook_dir }}/../../vars/" + tags: "deletecertspod" + +- name: Create k8s deployment files + template: + src: "{{ playbook_dir }}/../deploy_k8s/fabricsetup/templates/fabric-pod.j2" + dest: "{{ playbook_dir }}/../../vars/fabric-pod.yml" - name: Start fabric pods - command: "/opt/fabric/bin/kubectl --server {{ k8s_server }}:8080 create -f {{ fabricworkdir }}/run/fabric-pod.yml" + command: "./kubectl --kubeconfig='kubeconfig' apply -f fabric-pod.yml" + args: + chdir: "{{ playbook_dir }}/../../vars/" tags: "fabricup" -- name: Locate a host to run peer channel create command - set_fact: - clihost: "{{ item }}" - clipeer: "{{ allpeers | random }}" - cliorderer: "{{ allorderers | random }}" - when: peers | length > 0 and clihost == "" - with_items: "{{ groups['allnodes'] }}" - -- name: Make sure that working directory exists and clean - file: - path: "{{ fabricworkdir }}/run/keyfiles/chaincode" - state: "directory" - mode: 0775 - when: clihost == inventory_hostname - -- name: Move chaincode to the server - copy: - src: "{{ playbook_dir }}/../deploy_k8s/fabricsetup/templates/firstcode.go" - dest: "{{ fabricworkdir }}/run/keyfiles/chaincode/firstcode.go" - mode: "u=rw,g=rw" - when: clihost == inventory_hostname - -- name: Create peer channel command script - template: - src: "{{ playbook_dir }}/../deploy_k8s/fabricsetup/templates/dochannel.j2" - dest: "{{ fabricworkdir }}/run/keyfiles/dochannel.sh" - mode: "u=rx,g=rx" - when: clihost == inventory_hostname - - name: Create peer channel command k8s deployment file template: src: "{{ playbook_dir }}/../deploy_k8s/fabricsetup/templates/cli-k8s.j2" - dest: "{{ fabricworkdir }}/run/cli-k8s.yml" - when: clihost == inventory_hostname + dest: "{{ playbook_dir }}/../../vars/cli-k8s.yml" - name: Wait for containers to be ready pause: seconds: 30 - name: Start fabriccli pod - command: >- - /opt/fabric/bin/kubectl --server {{ k8s_server }}:8080 create - -f {{ fabricworkdir }}/run/cli-k8s.yml - when: clihost == inventory_hostname + command: "./kubectl --kubeconfig='kubeconfig' apply -f cli-k8s.yml" + args: + chdir: "{{ playbook_dir }}/../../vars/" tags: "cliup" diff --git a/src/agent/ansible/roles/deploy_k8s/fabricsetup/tasks/destroy.yml b/src/agent/ansible/roles/deploy_k8s/fabricsetup/tasks/destroy.yml index 7adff3fb..4a08031a 100755 --- a/src/agent/ansible/roles/deploy_k8s/fabricsetup/tasks/destroy.yml +++ b/src/agent/ansible/roles/deploy_k8s/fabricsetup/tasks/destroy.yml @@ -2,33 +2,22 @@ - name: Setup variables set_fact: fabricworkdir: "/opt/gopath/{{ env }}/fabric" - k8s_server: "{{ hostvars[groups['allnodes'][0]].private_ip }}" - clihost: "{{ groups['allnodes'][0] }}" - name: Stop fabric pods command: >- - /opt/fabric/bin/kubectl --server {{ k8s_server }}:8080 delete - -f {{ fabricworkdir }}/run/fabric-pod.yml + ./kubectl --kubeconfig='kubeconfig' delete -f fabric-pod.yml + args: + chdir: "{{ playbook_dir }}/../../vars/" tags: "fabricdown" - name: Stop fabriccli pod command: >- - /opt/fabric/bin/kubectl --server {{ k8s_server }}:8080 delete - -f {{ fabricworkdir }}/run/cli-k8s.yml - when: clihost == inventory_hostname + ./kubectl --kubeconfig='kubeconfig' delete -f cli-k8s.yml + args: + chdir: "{{ playbook_dir }}/../../vars/" tags: "clidown" -- name: Test if there are any chaincode container images - shell: docker images | grep "dev-*" | cat | awk '{print $1}' - register: images - no_log: true - -- name: Remove these chaincode container images - shell: docker images | grep "dev-*" | cat | awk '{print $1}' | xargs docker rmi -f - when: images.stdout != "" - no_log: true - - name: Make sure that working directory is removed file: - path: "{{ fabricworkdir }}/run" + path: "{{ fabricworkdir }}" state: absent diff --git a/src/agent/ansible/roles/deploy_k8s/fabricsetup/templates/certssetup.j2 b/src/agent/ansible/roles/deploy_k8s/fabricsetup/templates/certssetup.j2 new file mode 100755 index 00000000..dc6395ed --- /dev/null +++ b/src/agent/ansible/roles/deploy_k8s/fabricsetup/templates/certssetup.j2 @@ -0,0 +1,19 @@ +--- +kind: Pod +apiVersion: v1 +metadata: + name: fabriccertspod +spec: + volumes: + - name: task-pv-storage + persistentVolumeClaim: + claimName: fabriccerts + containers: + - name: task-pv-container + image: busybox + args: + - sleep + - "3000" + volumeMounts: + - mountPath: "/fabriccerts" + name: task-pv-storage diff --git a/src/agent/ansible/roles/deploy_k8s/fabricsetup/templates/cli-k8s.j2 b/src/agent/ansible/roles/deploy_k8s/fabricsetup/templates/cli-k8s.j2 index fc69371a..5bf28ae7 100755 --- a/src/agent/ansible/roles/deploy_k8s/fabricsetup/templates/cli-k8s.j2 +++ b/src/agent/ansible/roles/deploy_k8s/fabricsetup/templates/cli-k8s.j2 @@ -6,21 +6,17 @@ metadata: labels: k8s-app: fabriccli spec: - nodeName: {{ inter_name }} restartPolicy: OnFailure volumes: - name: varrun hostPath: path: /var/run - - name: keyfiles - hostPath: - path: {{ fabricworkdir }}/run/keyfiles - - name: chaincode - hostPath: - path: {{ fabricworkdir }}/run/keyfiles/chaincode + - name: task-pv-storage + persistentVolumeClaim: + claimName: fabriccerts containers: - name: fabriccli - image: {{ fabric.repo.url }}fabric-tools:{{ thetag }} + image: {{ fabric.repo.url }}fabric-tools:{{ fabric.baseimage_tag }} imagePullPolicy: IfNotPresent securityContext: privileged: true @@ -32,8 +28,6 @@ spec: volumeMounts: - mountPath: /host/var/run name: varrun - - mountPath: /opt/gopath/src/chaincode - name: chaincode - mountPath: /etc/hyperledger/allorgs - name: keyfiles + name: task-pv-storage command: ["/etc/hyperledger/allorgs/dochannel.sh"] diff --git a/src/agent/ansible/roles/deploy_k8s/fabricsetup/templates/dochannel.j2 b/src/agent/ansible/roles/deploy_k8s/fabricsetup/templates/dochannel.j2 index d3b4dffa..34500ead 100755 --- a/src/agent/ansible/roles/deploy_k8s/fabricsetup/templates/dochannel.j2 +++ b/src/agent/ansible/roles/deploy_k8s/fabricsetup/templates/dochannel.j2 @@ -3,35 +3,36 @@ # Set up environment varilables first {% if fabric.tls %} -export CORE_PEER_TLS_KEY_FILE=/etc/hyperledger/allorgs/{{ clipeer.org }}/peers/{{ clipeer.name }}.{{ clipeer.org }}/tls/server.key -export CORE_PEER_TLS_ROOTCERT_FILE=/etc/hyperledger/allorgs/{{ clipeer.org }}/peers/{{ clipeer.name }}.{{ clipeer.org }}/tls/ca.crt +export CORE_PEER_TLS_KEY_FILE=/etc/hyperledger/allorgs/keyfiles/{{ clipeer.org }}/peers/{{ clipeer.name }}.{{ clipeer.org }}/tls/server.key +export CORE_PEER_TLS_ROOTCERT_FILE=/etc/hyperledger/allorgs/keyfiles/{{ clipeer.org }}/peers/{{ clipeer.name }}.{{ clipeer.org }}/tls/ca.crt export CORE_PEER_ID={{ clipeer.name }} export CORE_PEER_ADDRESS={{ clipeer.name }}:7051 export CORE_PEER_LISTENADDRESS={{ clipeer.name }}:7051 export CORE_PEER_CHAINCODELISTENADDRESS={{ clipeer.name }}:7052 export CORE_PEER_LOCALMSPID={{ clipeer.org }} -export CORE_PEER_MSPCONFIGPATH=/etc/hyperledger/allorgs/{{ clipeer.org }}/users/Admin@{{ clipeer.org }}/msp +export CORE_PEER_MSPCONFIGPATH=/etc/hyperledger/allorgs/keyfiles/{{ clipeer.org }}/users/Admin@{{ clipeer.org }}/msp # Run the peer channel create command -cd /etc/hyperledger/allorgs +cd /etc/hyperledger/allorgs/keyfiles peer channel create -o {{ cliorderer.name }}:7050 -c firstchannel \ - -f /etc/hyperledger/allorgs/firstchannel.tx --tls true --timeout 240 \ - --cafile /etc/hyperledger/allorgs/{{ cliorderer.org }}/orderers/{{ cliorderer.name }}.{{ cliorderer.org }}/msp/tlscacerts/tlsca.{{ cliorderer.org }}-cert.pem + -f /etc/hyperledger/allorgs/keyfiles/firstchannel.tx --tls true --timeout 240 \ + --cafile /etc/hyperledger/allorgs/keyfiles/{{ cliorderer.org }}/orderers/{{ cliorderer.name }}.{{ cliorderer.org }}/msp/tlscacerts/tlsca.{{ cliorderer.org }}-cert.pem export CORE_PEER_TLS_ENABLED=true export GOPATH=/opt/gopath mkdir -p $GOPATH/src/chaincode +cp firstcode.go $GOPATH/src/chaincode {% for peer in allpeers %} -export CORE_PEER_TLS_KEY_FILE=/etc/hyperledger/allorgs/{{ peer.org }}/peers/{{ peer.name }}.{{ peer.org }}/tls/server.key -export CORE_PEER_TLS_ROOTCERT_FILE=/etc/hyperledger/allorgs/{{ peer.org }}/peers/{{ peer.name }}.{{ peer.org }}/tls/ca.crt +export CORE_PEER_TLS_KEY_FILE=/etc/hyperledger/allorgs/keyfiles/{{ peer.org }}/peers/{{ peer.name }}.{{ peer.org }}/tls/server.key +export CORE_PEER_TLS_ROOTCERT_FILE=/etc/hyperledger/allorgs/keyfiles/{{ peer.org }}/peers/{{ peer.name }}.{{ peer.org }}/tls/ca.crt export CORE_PEER_ID={{ peer.name }} export CORE_PEER_ADDRESS={{ peer.name }}:7051 export CORE_PEER_LISTENADDRESS={{ peer.name }}:7051 export CORE_PEER_CHAINCODELISTENADDRESS={{ peer.name }}:7052 export CORE_PEER_LOCALMSPID={{ peer.org }} -export CORE_PEER_MSPCONFIGPATH=/etc/hyperledger/allorgs/{{ peer.org }}/users/Admin@{{ peer.org }}/msp +export CORE_PEER_MSPCONFIGPATH=/etc/hyperledger/allorgs/keyfiles/{{ peer.org }}/users/Admin@{{ peer.org }}/msp peer channel join -b firstchannel.block @@ -43,17 +44,17 @@ peer chaincode install -n firstchaincode -v 1.0 -p chaincode echo '-------------- Instantiate chaincode' {% set chainpeer = peers[0] %} -export CORE_PEER_TLS_KEY_FILE=/etc/hyperledger/allorgs/{{ chainpeer.org }}/peers/{{ chainpeer.name }}.{{ chainpeer.org }}/tls/server.key -export CORE_PEER_TLS_ROOTCERT_FILE=/etc/hyperledger/allorgs/{{ chainpeer.org }}/peers/{{ chainpeer.name }}.{{ chainpeer.org }}/tls/ca.crt +export CORE_PEER_TLS_KEY_FILE=/etc/hyperledger/allorgs/keyfiles/{{ chainpeer.org }}/peers/{{ chainpeer.name }}.{{ chainpeer.org }}/tls/server.key +export CORE_PEER_TLS_ROOTCERT_FILE=/etc/hyperledger/allorgs/keyfiles/{{ chainpeer.org }}/peers/{{ chainpeer.name }}.{{ chainpeer.org }}/tls/ca.crt export CORE_PEER_ID={{ chainpeer.name }} export CORE_PEER_ADDRESS={{ chainpeer.name }}:7051 export CORE_PEER_LISTENADDRESS={{ chainpeer.name }}:7051 export CORE_PEER_CHAINCODELISTENADDRESS={{ chainpeer.name }}:7052 export CORE_PEER_LOCALMSPID={{ chainpeer.org }} -export CORE_PEER_MSPCONFIGPATH=/etc/hyperledger/allorgs/{{ chainpeer.org }}/users/Admin@{{ chainpeer.org }}/msp +export CORE_PEER_MSPCONFIGPATH=/etc/hyperledger/allorgs/keyfiles/{{ chainpeer.org }}/users/Admin@{{ chainpeer.org }}/msp peer chaincode instantiate -o {{ cliorderer.name }}:7050 --tls true \ - --cafile /etc/hyperledger/allorgs/{{ cliorderer.org }}/orderers/{{ cliorderer.name }}.{{ cliorderer.org }}/msp/tlscacerts/tlsca.{{ cliorderer.org }}-cert.pem \ + --cafile /etc/hyperledger/allorgs/keyfiles/{{ cliorderer.org }}/orderers/{{ cliorderer.name }}.{{ cliorderer.org }}/msp/tlscacerts/tlsca.{{ cliorderer.org }}-cert.pem \ -C firstchannel -n firstchaincode -v 1.0 -c '{"Args":["init","a", "100", "b","200"]}' -P "OR ('{{ orgmembers }}.member')" # Query the chaincode @@ -67,16 +68,17 @@ export CORE_PEER_ADDRESS={{ clipeer.name }}:7051 export CORE_PEER_LISTENADDRESS={{ clipeer.name }}:7051 export CORE_PEER_CHAINCODELISTENADDRESS={{ clipeer.name }}:7052 export CORE_PEER_LOCALMSPID={{ clipeer.org }} -export CORE_PEER_MSPCONFIGPATH=/etc/hyperledger/allorgs/{{ clipeer.org }}/users/Admin@{{ clipeer.org }}/msp +export CORE_PEER_MSPCONFIGPATH=/etc/hyperledger/allorgs/keyfiles/{{ clipeer.org }}/users/Admin@{{ clipeer.org }}/msp # Run the peer channel create command -cd /etc/hyperledger/allorgs +cd /etc/hyperledger/allorgs/keyfiles peer channel create -o {{ cliorderer.name }}:7050 -c firstchannel \ - -f /etc/hyperledger/allorgs/firstchannel.tx --timeout 240 + -f /etc/hyperledger/allorgs/keyfiles/firstchannel.tx --timeout 240 export CORE_PEER_TLS_ENABLED=false export GOPATH=/opt/gopath mkdir -p $GOPATH/src/chaincode +cp ../firstcode.go $GOPATH/src/chaincode {% for peer in allpeers %} @@ -86,7 +88,7 @@ export CORE_PEER_LISTENADDRESS={{ peer.name }}:7051 export CORE_PEER_CHAINCODELISTENADDRESS={{ peer.name }}:7052 export CORE_PEER_LOCALMSPID={{ peer.org }} -export CORE_PEER_MSPCONFIGPATH=/etc/hyperledger/allorgs/{{ peer.org }}/users/Admin@{{ peer.org }}/msp +export CORE_PEER_MSPCONFIGPATH=/etc/hyperledger/allorgs/keyfiles/{{ peer.org }}/users/Admin@{{ peer.org }}/msp peer channel join -b firstchannel.block @@ -97,15 +99,15 @@ peer chaincode install -n firstchaincode -v 1.0 -p chaincode # Instantiate the chaincode echo '-------------- Instantiate chaincode' -{% set chainpeer = peers[0] %} +{% set chainpeer = allpeers[0] %} export CORE_PEER_ID={{ chainpeer.name }} export CORE_PEER_ADDRESS={{ chainpeer.name }}:7051 export CORE_PEER_LISTENADDRESS={{ chainpeer.name }}:7051 export CORE_PEER_CHAINCODELISTENADDRESS={{ chainpeer.name }}:7052 export CORE_PEER_LOCALMSPID={{ chainpeer.org }} -export CORE_PEER_MSPCONFIGPATH=/etc/hyperledger/allorgs/{{ chainpeer.org }}/users/Admin@{{ chainpeer.org }}/msp +export CORE_PEER_MSPCONFIGPATH=/etc/hyperledger/allorgs/keyfiles/{{ chainpeer.org }}/users/Admin@{{ chainpeer.org }}/msp peer chaincode instantiate -o {{ cliorderer.name }}:7050 \ -C firstchannel -n firstchaincode -v 1.0 -c '{"Args":["init","a", "100", "b","200"]}' -P "OR ('{{ orgmembers }}.member')" -{% endif %} \ No newline at end of file +{% endif %} diff --git a/src/agent/ansible/roles/deploy_k8s/fabricsetup/templates/fabric-pod.j2 b/src/agent/ansible/roles/deploy_k8s/fabricsetup/templates/fabric-pod.j2 index 06f1b3cb..b6cd3250 100755 --- a/src/agent/ansible/roles/deploy_k8s/fabricsetup/templates/fabric-pod.j2 +++ b/src/agent/ansible/roles/deploy_k8s/fabricsetup/templates/fabric-pod.j2 @@ -2,7 +2,7 @@ # # Example usage: kubectl create -f -{% for zookeeper in zookeepers %} +{% for zookeeper in zookeeperls %} --- apiVersion: v1 kind: Pod @@ -11,11 +11,10 @@ metadata: labels: k8s-app: {{ zookeeper }} spec: - nodeName: {{ inter_name }} restartPolicy: Always containers: - name: {{ zookeeper }} - image: {{ fabric.repo.url }}fabric-zookeeper:{{ helpertag }} + image: {{ fabric.repo.url }}fabric-zookeeper:{{ fabric.helper_tag }} imagePullPolicy: IfNotPresent env: - { name: "ZOO_MY_ID", value: "{{ zoo_ids[zookeeper] }}" } @@ -40,7 +39,7 @@ spec: - { name: "port3", port: 2181 } {% endfor %} -{% for kafka in kafkas %} +{% for kafka in kafkals %} --- apiVersion: v1 kind: Pod @@ -49,11 +48,10 @@ metadata: labels: k8s-app: {{ kafka }} spec: - nodeName: {{ inter_name }} restartPolicy: Always containers: - name: {{ kafka }} - image: {{ fabric.repo.url }}fabric-kafka:{{ helpertag }} + image: {{ fabric.repo.url }}fabric-kafka:{{ fabric.helper_tag }} imagePullPolicy: IfNotPresent env: - { name: "KAFKA_MESSAGE_MAX_BYTES", value: "103809024" } @@ -79,7 +77,7 @@ spec: - { name: "port2", port: 9093 } {% endfor %} -{% for ca in cas %} +{% for ca in allcas %} --- apiVersion: v1 kind: Pod @@ -88,28 +86,27 @@ metadata: labels: k8s-app: {{ ca.name }} spec: - nodeName: {{ inter_name }} restartPolicy: Always volumes: - - name: keyfiles - hostPath: - path: {{ fabricworkdir }}/run/keyfiles/{{ ca.org }} + - name: task-pv-storage + persistentVolumeClaim: + claimName: fabriccerts containers: - name: {{ ca.name }} - image: {{ fabric.repo.url }}fabric-ca:{{ fabric.ca.image_tag | default(thetag) }} + image: {{ fabric.repo.url }}fabric-ca:{{ fabric.ca.image_tag }} imagePullPolicy: IfNotPresent env: - - { name: "FABRIC_CA_HOME", value: "/etc/hyperledger/fabric-ca-server-config/ca" } + - { name: "FABRIC_CA_HOME", value: "/etc/hyperledger/fabric-ca-server-config//keyfiles/{{ ca.org }}/ca" } - { name: "FABRIC_CA_SERVER_CA_NAME", value: "{{ ca.name}}" } - - { name: "FABRIC_CA_SERVER_CA_KEYFILE", value: "/etc/hyperledger/fabric-ca-server-config/ca/ca_private.key" } - - { name: "FABRIC_CA_SERVER_CA_CERTFILE", value: "/etc/hyperledger/fabric-ca-server-config/ca/ca.{{ ca.org }}-cert.pem" } + - { name: "FABRIC_CA_SERVER_CA_KEYFILE", value: "/etc/hyperledger/fabric-ca-server-config/keyfiles/{{ ca.org }}/ca/ca_private.key" } + - { name: "FABRIC_CA_SERVER_CA_CERTFILE", value: "/etc/hyperledger/fabric-ca-server-config/keyfiles/{{ ca.org }}/ca/ca.{{ ca.org }}-cert.pem" } {% if fabric.tls %} - { name: "FABRIC_CA_SERVER_TLS_ENABLED", value: "true" } - - { name: "FABRIC_CA_SERVER_TLS_KEYFILE", value: "/etc/hyperledger/fabric-ca-server-config/tlsca/tlsca_private.key" } - - { name: "FABRIC_CA_SERVER_TLS_CERTFILE", value: "/etc/hyperledger/fabric-ca-server-config/tlsca/tlsca.{{ ca.org }}-cert.pem" } + - { name: "FABRIC_CA_SERVER_TLS_KEYFILE", value: "/etc/hyperledger/fabric-ca-server-config/keyfiles/{{ ca.org }}/tlsca/tlsca_private.key" } + - { name: "FABRIC_CA_SERVER_TLS_CERTFILE", value: "/etc/hyperledger/fabric-ca-server-config/keyfiles/{{ ca.org }}/tlsca/tlsca.{{ ca.org }}-cert.pem" } {% endif %} volumeMounts: - - { mountPath: "/etc/hyperledger/fabric-ca-server-config", name: "keyfiles" } + - { mountPath: "/etc/hyperledger/fabric-ca-server-config", name: "task-pv-storage" } command: ["fabric-ca-server"] args: ["start", "-b", "{{ fabric.ca.admin }}:{{ fabric.ca.adminpw }}", "-d"] --- @@ -128,7 +125,7 @@ spec: {% endfor %} -{% for peer in peers %} +{% for peer in allpeers %} --- apiVersion: v1 kind: Pod @@ -137,33 +134,30 @@ metadata: labels: k8s-app: {{ peer.name }} spec: - nodeName: {{ inter_name }} restartPolicy: Always volumes: - name: varrun hostPath: path: /var/run - - name: keyfiles - hostPath: - path: {{ fabricworkdir }}/run/keyfiles - - name: peerorg - hostPath: - path: {{ fabricworkdir }}/run/keyfiles/{{ peer.org }}/peers/{{ peer.name }}.{{ peer.org }} + - name: task-pv-storage + persistentVolumeClaim: + claimName: fabriccerts containers: {% if fabric.peer_db == 'CouchDB' %} - name: couchdb-{{ peer.name }} - image: {{ fabric.repo.url }}fabric-couchdb:{{ helpertag }} + image: {{ fabric.repo.url }}fabric-couchdb:{{ fabric.helper_tag }} imagePullPolicy: IfNotPresent securityContext: privileged: true {% endif %} - name: {{ peer.name }} - image: {{ fabric.repo.url }}fabric-peer:{{ thetag }} + image: {{ fabric.repo.url }}fabric-peer:{{ fabric.baseimage_tag }} imagePullPolicy: IfNotPresent securityContext: privileged: true env: - { name: "CORE_VM_ENDPOINT", value: "unix:///host/var/run/docker.sock" } + - { name: "FABRIC_CFG_PATH", value: "/etc/hyperledger/fabric/keyfiles/{{ peer.org }}/peers/{{ peer.name }}.{{ peer.org }}" } - { name: "CORE_LOGGING_LEVEL", value: "DEBUG" } - { name: "CORE_PEER_ENDORSER_ENABLED", value: "true" } - { name: "CORE_PEER_GOSSIP_USELEADERELECTION", @@ -172,9 +166,9 @@ spec: - { name: "CORE_PEER_PROFILE_ENABLED", value: "true" } - { name: "CORE_PEER_TLS_ENABLED", value: "{{ fabric.tls | lower }}" } {% if fabric.tls %} - - { name: "CORE_PEER_TLS_CERT_FILE", value: "/etc/hyperledger/fabric/tls/server.crt" } - - { name: "CORE_PEER_TLS_KEY_FILE", value: "/etc/hyperledger/fabric/tls/server.key" } - - { name: "CORE_PEER_TLS_ROOTCERT_FILE", value: "/etc/hyperledger/fabric/tls/ca.crt" } + - { name: "CORE_PEER_TLS_CERT_FILE", value: "/etc/hyperledger/fabric/keyfiles/{{ peer.org }}/peers/{{ peer.name }}.{{ peer.org }}/tls/server.crt" } + - { name: "CORE_PEER_TLS_KEY_FILE", value: "/etc/hyperledger/fabric/keyfiles/{{ peer.org }}/peers/{{ peer.name }}.{{ peer.org }}/tls/server.key" } + - { name: "CORE_PEER_TLS_ROOTCERT_FILE", value: "/etc/hyperledger/fabric/keyfiles/{{ peer.org }}/peers/{{ peer.name }}.{{ peer.org }}/tls/ca.crt" } {% endif %} - { name: "CORE_PEER_ID", value: "{{ peer.name }}" } - { name: "CORE_PEER_GOSSIP_EXTERNALENDPOINT", value: "{{ peer.name }}:7051" } @@ -182,15 +176,14 @@ spec: - { name: "CORE_PEER_LISTENADDRESS", value: "{{ peer.name }}:7051" } - { name: "CORE_PEER_CHAINCODELISTENADDRESS", value: "{{ peer.name }}:7052" } - { name: "CORE_PEER_LOCALMSPID", value: "{{ peer.org }}" } - - { name: "CORE_PEER_MSPCONFIGPATH", value: "/etc/hyperledger/fabric/msp" } + - { name: "CORE_PEER_MSPCONFIGPATH", value: "/etc/hyperledger/fabric/keyfiles/{{ peer.org }}/peers/{{ peer.name }}.{{ peer.org }}/msp" } {% if fabric.peer_db == 'CouchDB' %} - { name: "CORE_LEDGER_STATE_STATEDATABASE", value: "CouchDB" } - { name: "CORE_LEDGER_STATE_COUCHDBCONFIG_COUCHDBADDRESS", value: "localhost:5984" } {% endif %} volumeMounts: - { mountPath: "/host/var/run", name: "varrun" } - - { mountPath: "/etc/hyperledger/fabric", name: "peerorg" } - - { mountPath: "/etc/hyperledger/allorgs", name: "keyfiles" } + - { mountPath: "/etc/hyperledger/fabric", name: "task-pv-storage" } command: ["peer"] args: ["node", "start"] --- @@ -210,7 +203,7 @@ spec: port: 7052 {% endfor %} -{% for orderer in orderers %} +{% for orderer in allorderers %} --- apiVersion: v1 kind: Pod @@ -219,39 +212,34 @@ metadata: labels: k8s-app: {{ orderer.name }} spec: - nodeName: {{ inter_name }} restartPolicy: Always volumes: - - name: keyfiles - hostPath: - path: {{ fabricworkdir }}/run/keyfiles - - name: ordererorg - hostPath: - path: {{ fabricworkdir }}/run/keyfiles/{{ orderer.org }}/orderers/{{ orderer.name }}.{{ orderer.org }} + - name: task-pv-storage + persistentVolumeClaim: + claimName: fabriccerts containers: - name: {{ orderer.name }} - image: {{ fabric.repo.url }}fabric-orderer:{{ thetag }} + image: {{ fabric.repo.url }}fabric-orderer:{{ fabric.baseimage_tag }} imagePullPolicy: IfNotPresent env: - { name: "ORDERER_GENERAL_LOGLEVEL", value: "debug" } - { name: "ORDERER_GENERAL_LISTENADDRESS", value: "0.0.0.0" } - { name: "ORDERER_GENERAL_GENESISMETHOD", value: "file" } - - { name: "ORDERER_GENERAL_GENESISFILE", value: "/var/hyperledger/allorgs/genesis.block" } + - { name: "ORDERER_GENERAL_GENESISFILE", value: "/var/hyperledger/orderer/keyfiles/genesis.block" } - { name: "ORDERER_GENERAL_LOCALMSPID", value: "{{ orderer.org }}" } - - { name: "ORDERER_GENERAL_LOCALMSPDIR", value: "/var/hyperledger/orderer/msp" } + - { name: "ORDERER_GENERAL_LOCALMSPDIR", value: "/var/hyperledger/orderer/keyfiles/{{ orderer.org }}/orderers/{{ orderer.name }}.{{ orderer.org }}/msp" } - { name: "ORDERER_GENERAL_TLS_CLIENTAUTHENABLED", value: "false" } - { name: "ORDERER_GENERAL_TLS_ENABLED", value: "{{ fabric.tls | lower }}" } {% if fabric.tls %} - - { name: "ORDERER_GENERAL_TLS_PRIVATEKEY", value: "/var/hyperledger/orderer/tls/server.key" } - - { name: "ORDERER_GENERAL_TLS_CERTIFICATE", value: "/var/hyperledger/orderer/tls/server.crt" } - - { name: "ORDERER_GENERAL_TLS_ROOTCAS", value: "[/var/hyperledger/orderer/tls/ca.crt]" } + - { name: "ORDERER_GENERAL_TLS_PRIVATEKEY", value: "/var/hyperledger/orderer/keyfiles/{{ orderer.org }}/orderers/{{ orderer.name }}.{{ orderer.org }}/tls/server.key" } + - { name: "ORDERER_GENERAL_TLS_CERTIFICATE", value: "/var/hyperledger/orderer/keyfiles/{{ orderer.org }}/orderers/{{ orderer.name }}.{{ orderer.org }}/tls/server.crt" } + - { name: "ORDERER_GENERAL_TLS_ROOTCAS", value: "[/var/hyperledger/orderer/keyfiles/{{ orderer.org }}/orderers/{{ orderer.name }}.{{ orderer.org }}/tls/ca.crt]" } {% endif %} - { name: "ORDERER_KAFKA_RETRY_SHORTINTERVAL", value: "1s" } - { name: "ORDERER_KAFKA_RETRY_SHORTTOTAL", value: "30s" } - { name: "ORDERER_KAFKA_VERBOSE", value: "true" } volumeMounts: - - { mountPath: "/var/hyperledger/allorgs", name: "keyfiles" } - - { mountPath: "/var/hyperledger/orderer", name: "ordererorg" } + - { mountPath: "/var/hyperledger/orderer", name: "task-pv-storage" } command: ["orderer"] --- kind: Service diff --git a/src/agent/ansible/roles/deploy_k8s/fabricsetup/templates/pvc.j2 b/src/agent/ansible/roles/deploy_k8s/fabricsetup/templates/pvc.j2 new file mode 100755 index 00000000..cd267ed2 --- /dev/null +++ b/src/agent/ansible/roles/deploy_k8s/fabricsetup/templates/pvc.j2 @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: fabriccerts +spec: + storageClassName: {{ storageclass }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi diff --git a/src/agent/ansible/roles/deploy_k8s/fabricsetup/templates/secrets.j2 b/src/agent/ansible/roles/deploy_k8s/fabricsetup/templates/secrets.j2 new file mode 100755 index 00000000..8537c86e --- /dev/null +++ b/src/agent/ansible/roles/deploy_k8s/fabricsetup/templates/secrets.j2 @@ -0,0 +1,12 @@ +{% for peer in allpeers %} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ peer.name }}.{{ peer.org}} +type: Opaque +data: + tls.ca.crt: YWRtaW4= + tls.server.crt: whatever + tls.server.key: whatever +{% endfor %} diff --git a/src/agent/ansible/roles/deploy_k8s/getbin/tasks/apply.yml b/src/agent/ansible/roles/deploy_k8s/getbin/tasks/apply.yml new file mode 100755 index 00000000..a54729ed --- /dev/null +++ b/src/agent/ansible/roles/deploy_k8s/getbin/tasks/apply.yml @@ -0,0 +1,30 @@ +--- +- name: Setup variables + set_fact: + fabricpath: "/opt/gopath/{{ env }}/src/github.com/hyperledger/fabric" + +- name: Setup hyperledger directory + file: + path: "{{ fabricpath }}/build" + state: directory + force: yes + mode: 0775 + +- stat: + path: "{{ fabricpath }}/build/bin/cryptogen" + register: binexists + +- name: Download fabric binaries + unarchive: + src: "{{ fabric.repo.bin }}" + dest: "{{ fabricpath }}/build" + remote_src: yes + when: fabric.repo.bin|length > 0 and binexists.stat.exists == false + tags: "downloadbin" + +- name: Download kubectl + get_url: + url: "https://storage.googleapis.com/kubernetes-release/release/v1.10.2/bin/linux/amd64/kubectl" + dest: "{{ playbook_dir }}/../../vars/kubectl" + mode: 0755 + tags: "downloadbin" diff --git a/src/agent/ansible/roles/deploy_k8s/getbin/tasks/destroy.yml b/src/agent/ansible/roles/deploy_k8s/getbin/tasks/destroy.yml new file mode 100755 index 00000000..849169c7 --- /dev/null +++ b/src/agent/ansible/roles/deploy_k8s/getbin/tasks/destroy.yml @@ -0,0 +1,3 @@ +--- + + diff --git a/src/agent/ansible/roles/deploy_k8s/plays.yml b/src/agent/ansible/roles/deploy_k8s/plays.yml index 3b444d93..c8776192 100755 --- a/src/agent/ansible/roles/deploy_k8s/plays.yml +++ b/src/agent/ansible/roles/deploy_k8s/plays.yml @@ -1,18 +1,20 @@ --- -- name: build fabric artifacts - hosts: builders - gather_facts: true - user: "{{ fabric.ssh_user }}" +- name: generate certificates + hosts: cloud + connection: local + become: true + become_user: root vars_files: - "./../../vars/{{ env }}.yml" tasks: - - include: "../deploy_compose/fabricbuild/tasks/{{ mode }}.yml" - tags: "fabricbuild" + - include: "getbin/tasks/{{ mode }}.yml" + tags: "getbin" - name: generate certificates - hosts: builders - gather_facts: false - user: "{{ fabric.ssh_user }}" + hosts: cloud + connection: local + become: true + become_user: root vars_files: - "./../../vars/{{ env }}.yml" tasks: @@ -20,9 +22,8 @@ tags: "certsetup" - name: setup fabric network - hosts: allnodes - gather_facts: false - user: "{{ fabric.ssh_user }}" + hosts: cloud + connection: local become: true become_user: root vars_files: diff --git a/src/agent/ansible/roles/env_k8s/master/tasks/apply.yml b/src/agent/ansible/roles/env_k8s/master/tasks/apply.yml index 1656bbd2..153badad 100755 --- a/src/agent/ansible/roles/env_k8s/master/tasks/apply.yml +++ b/src/agent/ansible/roles/env_k8s/master/tasks/apply.yml @@ -132,5 +132,4 @@ src: "/etc/kubernetes/kubeconfig" dest: "{{ playbook_dir }}/../../vars/kubeconfig" flat: yes - become: false - tags: "kubeconfig" \ No newline at end of file + tags: "kubeconfig"