heat_template_version: 2017-02-24

parameters:

  image_to_use: { type: string, label: "Image name or ID" }

  flavor_to_use: { type: string, label: "Flavor name" }

  key_name_user: { type: string, label: "User ssh public key" }

  root_pw: { type: string, label: "Root password" }

  avail_zone: { type: string, label: "Availability Zone", default: "nova" }

  host_prefix: { type: string, label: "Hostname prefix", default: "bigdata-test" }

  host_name: { type: string, label: "Server Hostname", default: "undefined" }

  lan_net_id: { type: string, label: "LAN Network ID", default: "undefined" }
  
  lan_subnet_name: { type: string, label: "LAN Sub network", default: "undefined" }

  lan_net_ippref: { type: string, label: "LAN IPv4 prefix" }

  wan_net_id: { type: string, label: "WAN Network ID", default: "undefined" }

  wan_subnet_name: { type: string, label: "WAN Sub network", default: "undefined" }

  wan_net_ippref: { type: string, label: "WAN IPv4 prefix", default: "undefined" }

  nameserver_list: { type: string, label: "Name server ip list" }

  sec_group_id: { type: string, label: "Security Group ID" }

  node_type: { type: string, label: "Node type", default: "generic" }

  node_id: { type: number, label: "Node ID" }

  kafka_ids: { type: string, label: "Kafka host id", default: "undefined" }

  hdfs_ids: { type: string, label: "Hadoop host id", default: "undefined" }

  spark_ids: { type: string, label: "Spark host id", default: "undefined" }

  wn_ids: { type: string, label: "Generic host id", default: "undefined" }

  admin_id: { type: string, label: "Administrator ID", default: "undefined" }

  fip_id: { type: string, label: "Floating point ID", default: "undefined" }

  oidc_client_id: { type: string, label: "OpenID Connect Client ID", default: "undefined" }

  oidc_client_secret: { type: string, label: "OpenID Connect Client secret", default: "undefined" }

  oidc_idp_url: { type: string, label: "OpenID Connect IdP metadata", default: "undefined" }

  mount_points: { type: string, label: "OpenID Connect IdP metadata", default: "undefined" }

  kafka_config: { type: json, label: "Configuration parameters for Kafka", default: {} }

  nifi_config: { type: json, label: "Configuration parameters for NiFi", default: {} }

conditions:

  public_access: { not: { equals: [ { get_param: wan_net_id }, "undefined" ] } }

  internal_access: { not: { equals: [ { get_param: lan_net_id }, "undefined" ] } }


resources:

  internal_port:
    type: OS::Neutron::Port
    condition: internal_access
    properties:
      name: { str_replace: { template: "internal-port-<%node_id%>", params: { <%node_id%>: { get_param: node_id } } } }
      network_id: { get_param: lan_net_id }
      fixed_ips:
        - {
            subnet: { get_param: lan_subnet_name },
            ip_address: {
              str_replace: {
                template: "<%ippref%>.<%node_id%>",
                params: { <%ippref%>: { get_param: lan_net_ippref }, <%node_id%>: { get_param: node_id } }
              }
            }
          }
      security_groups: [ { get_param: sec_group_id }, ]


  public_port:
    type: OS::Neutron::Port
    condition: public_access
    properties:
      name: { str_replace: { template: "public-port-<%node_id%>", params: { <%node_id%>: { get_param: node_id } } } }
      network_id: { get_param: wan_net_id }
      fixed_ips:
        - {
            subnet: { get_param: wan_subnet_name },
            ip_address: {
                str_replace: {
                  template: "<%ippref%>.<%node_id%>",
                  params: { <%ippref%>: { get_param: wan_net_ippref }, <%node_id%>: { get_param: node_id } }
                }
            }
          }
      security_groups: [ { get_param: sec_group_id }, ]

  public_fip_ass:
    type: OS::Neutron::FloatingIPAssociation
    condition: public_access
    properties:
      floatingip_id: { get_param: fip_id }
      port_id: { get_resource: public_port }

  server_instance:
    type: OS::Nova::Server
    properties:
      name: {
        str_replace: {
          template: "<%host_pref%>-<%node_id%>",
          params: { <%host_pref%> : { get_param: host_prefix }, <%node_id%>: { get_param: node_id } }
        }
      }
      key_name: { get_param: key_name_user }
      image: { get_param: image_to_use }
      flavor: { get_param: flavor_to_use }
      admin_pass: { get_param: root_pw }
      networks:
        - port: { if: [ "internal_access", { get_resource: internal_port }, { get_resource: public_port } ] }
      user_data_format: RAW
      user_data:
        str_replace:
          template: |
            #!/bin/bash
            yum -q -y install epel-release yum-priorities vim-enhanced wget java-1.8.0-openjdk
            wget -O /usr/libexec/cloudveneto_node_setup http://artifacts.pd.infn.it/templates/SMACT/heat/node_setup.py
            chmod u+x /usr/libexec/cloudveneto_node_setup
            mkdir 600 /etc/cloudveneto
            cat > /etc/cloudveneto/node_setup.conf << EOF
            [main]
            host_prefix: <%host_pref%>
            node_id: <%node_id%>
            lan_ippre: <%lan_ippre%>
            nameservers: <%ns_list%>
            node_type: <%node_type%>
            rootpwd: <%root_pwd%>
            admin_id: <%admin_id%>
            oidc_id: <%oidc_id%>
            oidc_sec: <%oidc_sec%>
            oidc_idp: <%oidc_idp%>
            servername: <%host_name%>
            kafka_ids: <%kafka_ids%>
            spark_ids: <%spark_ids%>
            hdfs_ids: <%hdfs_ids%>
            wn_ids: <%wn_ids%>
            mount_points: <%mount_p%>
            EOF
            cat > /etc/cloudveneto/nifi_config.json << EOF
            <%nifi_cfg%>
            EOF
            cat > /etc/cloudveneto/kafka_config.json << EOF
            <%kafka_cfg%>
            EOF
            /usr/libexec/cloudveneto_node_setup
          params:
            <%host_pref%>: { get_param: host_prefix }
            <%host_name%>: { get_param: host_name }
            <%lan_ippre%>: { get_param: lan_net_ippref }
            <%node_id%>:   { get_param: node_id }
            <%kafka_ids%>: { get_param: kafka_ids }
            <%hdfs_ids%>:  { get_param: hdfs_ids }
            <%spark_ids%>: { get_param: spark_ids }
            <%wn_ids%>:    { get_param: wn_ids }
            <%ns_list%>:   { get_param: nameserver_list }
            <%node_type%>: { get_param: node_type }
            <%root_pwd%>:  { get_param: root_pw }
            <%admin_id%>:  { get_param: admin_id }
            <%oidc_id%>:   { get_param: oidc_client_id }
            <%oidc_sec%>:  { get_param: oidc_client_secret }
            <%oidc_idp%>:  { get_param: oidc_idp_url }
            <%mount_p%>:   { get_param: mount_points }
            <%kafka_cfg%>: { get_param: kafka_config }
            <%nifi_cfg%>:  { get_param: nifi_config }

outputs:

  instance_id:
    description: ID of the server instance
    value: { get_resource: server_instance }

  instance_intport:
    description: Internal port
    value: { get_resource: internal_port }
    condition: internal_access

  instance_extport:
    description: External port
    value: { get_resource: public_port }
    condition: public_access

#
# Post-inst commands:
#   ssh-keyscan -t rsa1,rsa,dsa -f /etc/ssh/shosts.equiv >> /etc/ssh/ssh_known_hosts
#   systemctl restart sshd
#
#   systemctl start zookeeper
#   systemctl start kafka
#
# (bigdatausr on master) start-dfs.sh
# (bigdatausr on master) start-master.sh
# (bigdatausr on master) start-slaves.sh
#
# https://www.linode.com/docs/databases/hadoop/how-to-install-and-set-up-hadoop-cluster/
# https://pandas.pydata.org/
# http://www.numpy.org/
# https://github.com/cerndb/dist-keras
# https://jaceklaskowski.gitbooks.io/mastering-apache-spark/spark-rdd-partitions.html
#
# latest package for spark-root in maven central: "org.diana-hep:spark-root_2.11:0.1.16"
# python call:
#   spark.read.format("org.dianahep.sparkroot.experimental").load("hdfs:///path/to/file.root")
#
# Simple tests
#   kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic mytopic
#   kafka-console-producer.sh --broker-list localhost:9092 --topic mytopic
#   kafka-topics.sh --list --zookeeper localhost:2181
#   kafka-console-consumer.sh --bootstrap-server localhost:9092 --from-beginning --topic mytopic
#
# Nifi post-inst operations:
# Install the keystore in /opt/bigdata/nifi/conf/service.jks:
#   the keystore pwd MUST BE equal to the root_pwd
#   the owner MUST BE bigdatausr.bigdatausr
# Run "/opt/bigdata/nifi/bin/nifi.sh start" as bigdatausr
#







