diff --git a/README.md b/README.md index ab1d858c..bfa033d3 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ For releases and release notes, see the [Releases](https://github.com/yetanalyti - [Metrics](doc/metrics.md) - [Docker Container](doc/docker.md) - [Demo](doc/demo.md) +- [Sample AWS Deployment](doc/aws.md) ## Contribution diff --git a/dev-resources/templates/lrspipe_ec2.yml b/dev-resources/templates/lrspipe_ec2.yml new file mode 100644 index 00000000..ecd23027 --- /dev/null +++ b/dev-resources/templates/lrspipe_ec2.yml @@ -0,0 +1,347 @@ +AWSTemplateFormatVersion: '2010-09-09' +Description: 'LRSPipe Application Template' +Parameters: + # Networking + VPCId: + Description: 'VPC on which to run LRSPipe' + Type: AWS::EC2::VPC::Id + LrsPipeSubnet: + Description: Subnet on which to run LRSPipe Server + Type: AWS::EC2::Subnet::Id + + # Pipe Server + InstanceType: + Type: String + Description: EC2 Instance Type to launch. + Default: c5.large + AllowedValues: + - c5.large + - c5.xlarge + - c5.2xlarge + - c5.4xlarge + InstanceAmiId: + Description: AMI (image) to use on EC2 instances. + Type: AWS::SSM::Parameter::Value + Default: '/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2' + InstanceKeyName: + Description: Name of an EC2 key pair that can be used to connect to the server (optional) + Type: String + InstanceSSHCidr: + Description: CIDR Range for SSH Access to instances + Type: String + Default: '173.147.0.0/16' + LrsPipeConfig: + Description: JSON Configuration of the LRSPipe Job + Type: String + LrsPipeVersion: + Description: Version of LRSPipe to download and install (public release versions on GitHub) + Type: String + Default: v0.0.15 + + LogRetentionInDays: + Description: How long to retain Cloudwatch Logs + Type: Number + Default: 7 + + +Conditions: + InstanceKeyNameProvided: + !Not [!Equals [!Ref InstanceKeyName, '']] + +Resources: + + # Logs + + LogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub '/yet/lrspipe/${AWS::StackName}' + RetentionInDays: !Ref LogRetentionInDays + + # Security + + PipeInstanceSG: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Enable SSH access via port 22 + VpcId: !Ref VPCId + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !Ref InstanceSSHCidr + Description: 'ssh from allowed IP' + + # Policy + InstancePolicy: + Type: AWS::IAM::ManagedPolicy + Description: IAM Policy for an Instance. + Properties: + ManagedPolicyName: !Sub '${AWS::StackName}-${AWS::Region}-lrspipe-instance-policy' + PolicyDocument: + Version: '2012-10-17' + Statement: + # Allow access to write to log group + - Effect: Allow + Action: + - 'logs:DescribeLogGroups' + Resource: + - !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:*' + - Effect: Allow + Action: + - 'logs:DescribeLogGroups' + - 'logs:CreateLogStream' + - 'logs:DescribeLogStreams' + - 'logs:PutLogEvents' + Resource: + - !GetAtt LogGroup.Arn + + + InstanceRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Effect: Allow + Principal: + Service: [ec2.amazonaws.com] + Action: ['sts:AssumeRole'] + Path: / + ManagedPolicyArns: + - !Ref InstancePolicy + - arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess # allow x-ray writes + + InstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Path: / + Roles: [!Ref InstanceRole] + + # Server + PipeInstance: + Type: AWS::EC2::Instance + Metadata: + AWS::CloudFormation::Init: + configSets: + default: + - 00_installPipe + - 01_setupCfnHup + - 02_configureCloudwatch + - 03_restartCloudwatch + - 04_configurePipe + - 05_enablePipe + - 06_startPipe + UpdateConfig: + - 04_configurePipe + - 07_restartPipe + UpdateVersion: + - 08_stopPipe + - 00_installPipe + - 06_startPipe + 00_installPipe: + files: + "/tmp/xapipe.zip": + source: !Join + - '/' + - - "https://github.com/yetanalytics/xapipe/releases/download" + - !Ref LrsPipeVersion + - "xapipe.zip" + mode: "000755" + owner: "root" + group: "root" + commands: + 00clear_previous_app: + cwd: "~" + command: "sudo rm -rf /opt/xapipe" + 01make_app_dir: + cwd: "~" + command: "sudo mkdir -p /opt/xapipe" + 02make_data_dir: + cwd: "~" + command: "sudo mkdir -p /var/lib/xapipe" + 03make_conf_dir: + cwd: "~" + command: "sudo mkdir -p /etc/xapipe" + 04unzip_pipe: + cwd: "~" + command: "sudo unzip /tmp/xapipe.zip -d /opt/xapipe/" + 01_setupCfnHup: + files: + '/etc/cfn/cfn-hup.conf': + content: !Sub | + [main] + stack=${AWS::StackId} + region=${AWS::Region} + interval=1 + mode: '000400' + owner: root + group: root + "/lib/systemd/system/cfn-hup.service": + content: !Sub | + [Unit] + Description=cfn-hup daemon + [Service] + Type=simple + ExecStart=/opt/aws/bin/cfn-hup + Restart=always + [Install] + WantedBy=multi-user.target + '/etc/cfn/hooks.d/xapipe-config-reloader.conf': + content: !Sub | + [cfn-version-update-hook] + triggers=post.update + path=Resources.PipeInstance.Metadata.AWS::CloudFormation::Init.04_configurePipe + action=/opt/aws/bin/cfn-init -v --stack ${AWS::StackId} --resource PipeInstance --region ${AWS::Region} --configsets UpdateConfig + runas=root + mode: '000400' + owner: root + group: root + '/etc/cfn/hooks.d/xapipe-version-reloader.conf': + content: !Sub | + [cfn-config-update-hook] + triggers=post.update + path=Resources.PipeInstance.Metadata.AWS::CloudFormation::Init.00_installPipe + action=/opt/aws/bin/cfn-init -v --stack ${AWS::StackId} --resource PipeInstance --region ${AWS::Region} --configsets UpdateVersion + runas=root + mode: '000400' + owner: root + group: root + commands: + 01enable_cfn_hup: + command: !Sub | + systemctl enable cfn-hup.service + 02start_cfn_hup: + command: !Sub | + systemctl start cfn-hup.service + 02_configureCloudwatch: + packages: + rpm: + amazon-cloudwatch-agent: https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm + files: + '/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json': + content: !Sub | + { + "metrics": { + "append_dimensions": { + "AutoScalingGroupName": "${!aws:AutoScalingGroupName}", + "ImageId": "${!aws:ImageId}", + "InstanceId": "${!aws:InstanceId}", + "InstanceType": "${!aws:InstanceType}" + }, + "metrics_collected": { + "cpu": { + "measurement": [ + "usage_active", + "usage_nice", + "usage_system", + "usage_user" + ] + }, + "mem": { + "measurement": [ + "mem_used_percent" + ] + }, + "swap": { + "measurement": [ + "swap_used_percent" + ] + } + } + }, + "logs":{ + "logs_collected": { + "files":{ + "collect_list": [ + { + "file_path": "/opt/xapipe/logs/*", + "auto_removal": true, + "log_group_name": "${LogGroup}", + "log_stream_name": "lrspipe-instance-{instance_id}" + } + ] + } + }, + "log_stream_name": "lrspipe-instance-{instance_id}" + } + } + # Invoke amazon-cloudwatch-agent-ctl to restart the AmazonCloudWatchAgent. + 03_restartCloudwatch: + commands: + 01_stop_service: + command: /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a stop + 02_start_service: + command: /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json -s + 04_configurePipe: + files: + '/etc/xapipe/job-conf.json': + content: !Ref LrsPipeConfig + mode: '000755' + owner: root + group: root + '/lib/systemd/system/xapipe.service': + content: !Sub | + [Unit] + Description=LRSPipe Service + [Service] + User=root + WorkingDirectory=/opt/xapipe + ExecStart=/opt/xapipe/bin/run.sh --json-file /etc/xapipe/job-conf.json --file-store-dir /var/lib/xapipe/db + SuccessExitStatus=143 + TimeoutStopSec=10 + Restart=on-failure + RestartSec=5 + [Install] + WantedBy=multi-user.target + mode: '000755' + owner: root + group: root + 05_enablePipe: + commands: + 01enable_pipe: + command: !Sub | + systemctl enable xapipe.service + 06_startPipe: + commands: + 01start_pipe: + command: !Sub | + systemctl start xapipe.service + 07_restartPipe: + commands: + 01daemon_reload: + command: | + systemctl daemon-reload + 02restart_pipe: + command: !Sub | + systemctl restart xapipe.service + 08_stopPipe: + commands: + 01stop_pipe: + command: | + systemctl stop xapipe.service + Properties: + ImageId: !Ref InstanceAmiId + InstanceType: !Ref InstanceType + KeyName: !If + - InstanceKeyNameProvided + - !Ref InstanceKeyName + - !Ref AWS::NoValue + SecurityGroupIds: + - !Ref PipeInstanceSG + IamInstanceProfile: !Ref InstanceProfile + SubnetId: !Ref LrsPipeSubnet + UserData: + Fn::Base64: !Sub | + #!/bin/bash -xe + echo 'Yet LRSPipe ${LrsPipeVersion}' + # run configsets + /opt/aws/bin/cfn-init -v --stack ${AWS::StackId} --resource PipeInstance --region ${AWS::Region} --configsets default + # signal CF + /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackId} --resource PipeInstance --region ${AWS::Region} + +Outputs: + LrsPipeSubmit: + Description: LRSPipe Subnet + Value: !Ref LrsPipeSubnet + Export: + Name: !Sub '${AWS::StackName}:LrsPipeSubnet' diff --git a/doc/aws.md b/doc/aws.md new file mode 100644 index 00000000..3a43e044 --- /dev/null +++ b/doc/aws.md @@ -0,0 +1,44 @@ +[<- Back to Index](index.md) + +# Example AWS Implementation + +This repo includes a sample AWS CloudFormation Template available at [`dev-resources/templates/lrspipe_ec2.yml`](https://github.com/yetanalytics/xapipe/blob/main/dev-resources/templates/lrspipe_ec2.yml). This template creates an EC2 Instance and launches an LRSPipe job on the server in Amazon Web Services. The template does not create the source or target LRS implementations and requires you to ensure that the Instance has proper access to both. In this section we will discuss how to deploy the LRSPipe template. + +__NOTE:__ *This configuration is not one-size-fits-all and you may require a different configuration for your particular needs. It is provided for demonstration purposes only and can be used as a reference to adapt to your particular enterprise's needs. If you apply these templates in your own AWS account, it can and will incur charges from Amazon Web Services. Yet Analytics is in no way responsible for any charges due to applying and implementing these templates, and is in no way responsible for any outcomes of applying these templates or implementing LRSPipe. If your team is interested in consulting or support in setting up or maintaining LRS Forwarding through LRSPipe or LRS infrastructure in general please [contact Yet here](https://www.sqllrs.com/contact)*. + +## Deployment + +Deploying the CloudFormation Template only requires a few straightforward steps. + +- Go to AWS CloudFormation Service +- Choose Create Stack (New Resources) +- Choose 'Template is Ready' / 'Upload a template file' +- Upload the Template `dev-resources/templates/lrspipe_ec2.yml` +- Click 'Next' + +### Configuration + +![LRSPipe Template Deployment Options](img/template-options.png) + +On the next page you will be presented with a number of deployment options. We will detail each one below: + +- `Stack Name`: Enter a name to identify this template. +- `InstanceAmiId`: This is the operating system image of the EC2 Instance. We do not recommend changing this from default as this is the setup that has been tested with this template. +- `InstanceKeyName`: If you would like to be able to SSH to the server for diagnostic purposes, enter the SSH key-pair name for the key you would like to use for authorization. If not leave it blank. +- `InstanceSSHCidr`: Provide a whitelisted CIDR Range for access to SSH to the instance. +- `InstanceType`: This is the size of the server to create. We recommend (and have tested) a c5.large for LRSPipe jobs, but you may want to go smaller or larger depending on use-case. +- `LogRetentionInDays`: This tells CloudWatch how long to hold onto application logs. This can affect costs stemming from large amounts of log storage. +- `LrsPipeConfig`: This is where you will paste in your LRSPipe JSON job configuration. This is where all of the actual job configuration takes place. See the [JSON-based Job Config](json.md) page for details and instructions on this step. +- `LrsPipeSubnet`: This field, in conjunction with the `VPCId` determine where in your network the LRSPipe instance is created. This is important as it may impact access to the source and target LRS'. If your LRS' are hosted in the same AWS account make sure this Subnet has access to them. Alternatively if your LRS' are hosted externally, make sure this Subnet has internet access configured. +- `LrsPipeVersion`: This is the version of LRSPipe software you want to deploy. This can be used to upgrade the version on a running instance as well. For a list of LRSPipe versions visit [releases](https://github.com/yetanalytics/xapipe/releases). +- `VPCId`: This field controls which Virtual Private Cloud the Instance resides in. Make sure that the chosen VPC contains the Subnet from that previous step. + +Now click 'Next' and proceed to deploy the template. + +## Monitoring + +![LRSPipe Template Deployment Options](img/resources.png) + +This template comes with a CloudWatch configuration that allows you to view the logs directly in AWS. Once the template is deployed, in order to find the logs, go to the resources tab as pictured below and click the `LogGroup` link. This will take you to the index of logs available for viewing. *Note: Keep in mind that CloudWatch has a slight publishing delay from the time that an event occurs in the system.* These logs can be used to ensure that the LRSPipe configuration was deployed successfully, or to debug issues that may arise. + +[<- Back to Index](index.md) diff --git a/doc/img/resources.png b/doc/img/resources.png new file mode 100644 index 00000000..3ea78034 Binary files /dev/null and b/doc/img/resources.png differ diff --git a/doc/img/template-options.png b/doc/img/template-options.png new file mode 100644 index 00000000..9b34c2d3 Binary files /dev/null and b/doc/img/template-options.png differ diff --git a/doc/index.md b/doc/index.md index be5ba6fc..2337b0e2 100644 --- a/doc/index.md +++ b/doc/index.md @@ -14,3 +14,4 @@ This process is one-way and any statements in the target LRS will not be replica - [Metrics](metrics.md) - [Docker Container](docker.md) - [Demo](demo.md) +- [Sample AWS Deployment](aws.md)