Merge pull request #58 from yetanalytics/cf_templates

PIP-61 - CF template

README.md

@@ -45,6 +45,7 @@ For releases and release notes, see the [Releases](https://github.com/yetanalyti
 - [Metrics](doc/metrics.md)
 - [Docker Container](doc/docker.md)
 - [Demo](doc/demo.md)
+- [Sample AWS Deployment](doc/aws.md)
 
 ## Contribution
 

dev-resources/templates/lrspipe_ec2.yml

@@ -0,0 +1,347 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Description: 'LRSPipe Application Template'
+Parameters:
+  # Networking
+  VPCId:
+    Description: 'VPC on which to run LRSPipe'
+    Type: AWS::EC2::VPC::Id
+  LrsPipeSubnet:
+    Description: Subnet on which to run LRSPipe Server
+    Type: AWS::EC2::Subnet::Id
+
+  # Pipe Server
+  InstanceType:
+    Type: String
+    Description: EC2 Instance Type to launch.
+    Default: c5.large
+    AllowedValues:
+      - c5.large
+      - c5.xlarge
+      - c5.2xlarge
+      - c5.4xlarge
+  InstanceAmiId:
+    Description: AMI (image) to use on EC2 instances.
+    Type: AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>
+    Default: '/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2'
+  InstanceKeyName:
+    Description: Name of an EC2 key pair that can be used to connect to the server (optional)
+    Type: String
+  InstanceSSHCidr:
+    Description: CIDR Range for SSH Access to instances
+    Type: String
+    Default: '173.147.0.0/16'
+  LrsPipeConfig:
+    Description: JSON Configuration of the LRSPipe Job
+    Type: String
+  LrsPipeVersion:
+    Description: Version of LRSPipe to download and install (public release versions on GitHub)
+    Type: String
+    Default: v0.0.15
+
+  LogRetentionInDays:
+    Description: How long to retain Cloudwatch Logs
+    Type: Number
+    Default: 7
+
+
+Conditions:
+  InstanceKeyNameProvided:
+    !Not [!Equals [!Ref InstanceKeyName, '']]
+
+Resources:
+
+  # Logs
+
+  LogGroup:
+    Type: AWS::Logs::LogGroup
+    Properties:
+      LogGroupName: !Sub '/yet/lrspipe/${AWS::StackName}'
+      RetentionInDays: !Ref LogRetentionInDays
+
+  # Security
+
+  PipeInstanceSG:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      GroupDescription: Enable SSH access via port 22
+      VpcId: !Ref VPCId
+      SecurityGroupIngress:
+      - IpProtocol: tcp
+        FromPort: 22
+        ToPort: 22
+        CidrIp: !Ref InstanceSSHCidr
+        Description: 'ssh from allowed IP'
+
+  # Policy
+  InstancePolicy:
+    Type: AWS::IAM::ManagedPolicy
+    Description: IAM Policy for an Instance.
+    Properties:
+      ManagedPolicyName: !Sub '${AWS::StackName}-${AWS::Region}-lrspipe-instance-policy'
+      PolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          # Allow access to write to log group
+          - Effect: Allow
+            Action:
+              - 'logs:DescribeLogGroups'
+            Resource:
+              - !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:*'
+          - Effect: Allow
+            Action:
+              - 'logs:DescribeLogGroups'
+              - 'logs:CreateLogStream'
+              - 'logs:DescribeLogStreams'
+              - 'logs:PutLogEvents'
+            Resource:
+              - !GetAtt LogGroup.Arn
+
+
+  InstanceRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Statement:
+        - Effect: Allow
+          Principal:
+            Service: [ec2.amazonaws.com]
+          Action: ['sts:AssumeRole']
+      Path: /
+      ManagedPolicyArns:
+        - !Ref InstancePolicy
+        - arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess # allow x-ray writes
+
+  InstanceProfile:
+    Type: AWS::IAM::InstanceProfile
+    Properties:
+      Path: /
+      Roles: [!Ref InstanceRole]
+
+  # Server
+  PipeInstance:
+    Type: AWS::EC2::Instance
+    Metadata:
+      AWS::CloudFormation::Init:
+        configSets:
+          default:
+            - 00_installPipe
+            - 01_setupCfnHup
+            - 02_configureCloudwatch
+            - 03_restartCloudwatch
+            - 04_configurePipe
+            - 05_enablePipe
+            - 06_startPipe
+          UpdateConfig:
+            - 04_configurePipe
+            - 07_restartPipe
+          UpdateVersion:
+            - 08_stopPipe
+            - 00_installPipe
+            - 06_startPipe
+        00_installPipe:
+          files:
+            "/tmp/xapipe.zip":
+              source: !Join
+                - '/'
+                - - "https://github.com/yetanalytics/xapipe/releases/download"
+                  - !Ref LrsPipeVersion
+                  - "xapipe.zip"
+              mode: "000755"
+              owner: "root"
+              group: "root"
+          commands:
+            00clear_previous_app:
+              cwd: "~"
+              command: "sudo rm -rf /opt/xapipe"
+            01make_app_dir:
+              cwd: "~"
+              command: "sudo mkdir -p /opt/xapipe"
+            02make_data_dir:
+              cwd: "~"
+              command: "sudo mkdir -p /var/lib/xapipe"
+            03make_conf_dir:
+              cwd: "~"
+              command: "sudo mkdir -p /etc/xapipe"
+            04unzip_pipe:
+              cwd: "~"
+              command: "sudo unzip /tmp/xapipe.zip -d /opt/xapipe/"
+        01_setupCfnHup:
+          files:
+            '/etc/cfn/cfn-hup.conf':
+              content: !Sub |
+                [main]
+                stack=${AWS::StackId}
+                region=${AWS::Region}
+                interval=1
+              mode: '000400'
+              owner: root
+              group: root
+            "/lib/systemd/system/cfn-hup.service":
+              content: !Sub |
+                [Unit]
+                Description=cfn-hup daemon
+                [Service]
+                Type=simple
+                ExecStart=/opt/aws/bin/cfn-hup
+                Restart=always
+                [Install]
+                WantedBy=multi-user.target
+            '/etc/cfn/hooks.d/xapipe-config-reloader.conf':
+              content: !Sub |
+                [cfn-version-update-hook]
+                triggers=post.update
+                path=Resources.PipeInstance.Metadata.AWS::CloudFormation::Init.04_configurePipe
+                action=/opt/aws/bin/cfn-init -v --stack ${AWS::StackId} --resource PipeInstance --region ${AWS::Region} --configsets UpdateConfig
+                runas=root
+              mode: '000400'
+              owner: root
+              group: root
+            '/etc/cfn/hooks.d/xapipe-version-reloader.conf':
+              content: !Sub |
+                [cfn-config-update-hook]
+                triggers=post.update
+                path=Resources.PipeInstance.Metadata.AWS::CloudFormation::Init.00_installPipe
+                action=/opt/aws/bin/cfn-init -v --stack ${AWS::StackId} --resource PipeInstance --region ${AWS::Region} --configsets UpdateVersion
+                runas=root
+              mode: '000400'
+              owner: root
+              group: root
+          commands:
+            01enable_cfn_hup:
+              command: !Sub |
+                systemctl enable cfn-hup.service
+            02start_cfn_hup:
+              command: !Sub |
+                systemctl start cfn-hup.service
+        02_configureCloudwatch:
+          packages:
+            rpm:
+              amazon-cloudwatch-agent: https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm
+          files:
+            '/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json':
+              content: !Sub |
+                {
+                  "metrics": {
+                    "append_dimensions": {
+                      "AutoScalingGroupName": "${!aws:AutoScalingGroupName}",
+                      "ImageId": "${!aws:ImageId}",
+                      "InstanceId": "${!aws:InstanceId}",
+                      "InstanceType": "${!aws:InstanceType}"
+                    },
+                    "metrics_collected": {
+                      "cpu": {
+                        "measurement": [
+                          "usage_active",
+                          "usage_nice",
+                          "usage_system",
+                          "usage_user"
+                        ]
+                      },
+                      "mem": {
+                        "measurement": [
+                          "mem_used_percent"
+                        ]
+                      },
+                      "swap": {
+                        "measurement": [
+                          "swap_used_percent"
+                        ]
+                      }
+                    }
+                  },
+                  "logs":{
+                    "logs_collected": {
+                      "files":{
+                        "collect_list": [
+                          {
+                            "file_path": "/opt/xapipe/logs/*",
+                            "auto_removal": true,
+                            "log_group_name": "${LogGroup}",
+                            "log_stream_name": "lrspipe-instance-{instance_id}"
+                          }
+                        ]
+                      }
+                    },
+                    "log_stream_name": "lrspipe-instance-{instance_id}"
+                  }
+                }
+        # Invoke amazon-cloudwatch-agent-ctl to restart the AmazonCloudWatchAgent.
+        03_restartCloudwatch:
+          commands:
+            01_stop_service:
+              command: /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop
+            02_start_service:
+              command: /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s
+        04_configurePipe:
+          files:
+            '/etc/xapipe/job-conf.json':
+              content: !Ref LrsPipeConfig
+              mode: '000755'
+              owner: root
+              group: root
+            '/lib/systemd/system/xapipe.service':
+              content: !Sub |
+                [Unit]
+                Description=LRSPipe Service
+                [Service]
+                User=root
+                WorkingDirectory=/opt/xapipe
+                ExecStart=/opt/xapipe/bin/run.sh --json-file /etc/xapipe/job-conf.json --file-store-dir /var/lib/xapipe/db
+                SuccessExitStatus=143
+                TimeoutStopSec=10
+                Restart=on-failure
+                RestartSec=5
+                [Install]
+                WantedBy=multi-user.target
+              mode: '000755'
+              owner: root
+              group: root
+        05_enablePipe:
+          commands:
+            01enable_pipe:
+              command: !Sub |
+                systemctl enable xapipe.service
+        06_startPipe:
+          commands:
+            01start_pipe:
+              command: !Sub |
+                systemctl start xapipe.service
+        07_restartPipe:
+          commands:
+            01daemon_reload:
+              command: |
+                systemctl daemon-reload
+            02restart_pipe:
+              command: !Sub |
+                systemctl restart xapipe.service
+        08_stopPipe:
+          commands:
+            01stop_pipe:
+              command: |
+                systemctl stop xapipe.service
+    Properties:
+      ImageId: !Ref InstanceAmiId
+      InstanceType: !Ref InstanceType
+      KeyName: !If
+        - InstanceKeyNameProvided
+        - !Ref InstanceKeyName
+        - !Ref AWS::NoValue
+      SecurityGroupIds:
+        - !Ref PipeInstanceSG
+      IamInstanceProfile: !Ref InstanceProfile
+      SubnetId: !Ref LrsPipeSubnet
+      UserData:
+        Fn::Base64: !Sub |
+          #!/bin/bash -xe
+          echo 'Yet LRSPipe ${LrsPipeVersion}'
+          # run configsets
+          /opt/aws/bin/cfn-init -v --stack ${AWS::StackId} --resource PipeInstance --region ${AWS::Region} --configsets default
+          # signal CF
+          /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackId} --resource PipeInstance --region ${AWS::Region}
+
+Outputs:
+  LrsPipeSubmit:
+    Description: LRSPipe Subnet
+    Value: !Ref LrsPipeSubnet
+    Export:
+      Name: !Sub '${AWS::StackName}:LrsPipeSubnet'