Skip to content

Commit

Permalink
Adding logic for creating and managing DAG invocations
Browse files Browse the repository at this point in the history 
Signed-off-by: Kway Yi Shen <[email protected]>
Signed-off-by: Kway Yi Shen <SamKway@localhost>
Signed-off-by: Kway Yi Shen <[email protected]>
  • Loading branch information
@NotAnAddictz
NotAnAddictz committed Aug 3, 2024
1 parent e5522c1 commit 486319d
Show file tree
Hide file tree
Showing 11 changed files with 357 additions and 50 deletions.
1 change: 1 addition & 0 deletions .github/configs/wordlist.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ Ctrl
currencyservice
Daglis
DAGMode
DAGTracePath
datacenter
Datacenter
dataflows
Expand Down
3 changes: 2 additions & 1 deletion cmd/config.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,5 +21,6 @@

"GRPCConnectionTimeoutSeconds": 15,
"GRPCFunctionTimeoutSeconds": 900,
"DAGMode": false
"DAGMode": false,
"DAGTracePath": "data/traces/sampled_150/20"
}
26 changes: 25 additions & 1 deletion cmd/loader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,12 @@ package main
import (
"flag"
"fmt"
"golang.org/x/exp/slices"
"os"
"os/exec"
"time"

"golang.org/x/exp/slices"

"github.com/vhive-serverless/loader/pkg/common"
"github.com/vhive-serverless/loader/pkg/config"
"github.com/vhive-serverless/loader/pkg/driver"
Expand Down Expand Up @@ -101,6 +103,14 @@ func main() {
log.Fatal("Unsupported platform! Supported platforms are [Knative, OpenWhisk, AWSLambda, Dirigent]")
}

if cfg.DAGMode {
cfg.TracePath = cfg.DAGTracePath
_, err := os.Stat(cfg.DAGTracePath)
if err != nil {
getTrace()
}
}

runTraceMode(&cfg, *iatGeneration, *generated)
}

Expand Down Expand Up @@ -188,3 +198,17 @@ func runTraceMode(cfg *config.LoaderConfiguration, iatOnly bool, generated bool)

experimentDriver.RunExperiment(iatOnly, generated)
}

func getTrace() bool {
_, err := exec.Command("git", "lfs", "pull").CombinedOutput()
if err != nil {
log.Warnf("Failed to get traces")
return false
}
_, err = exec.Command("tar", "-xzf", "data/traces/reference/sampled_150.tar.gz", "-C", "data/traces").CombinedOutput()
if err != nil {
log.Warnf("Failed to extract sample")
return false
}
return true
}
Binary file added data/traces/example/dag_structure.xlsx
View file
Binary file not shown.
3 changes: 2 additions & 1 deletion docs/configuration.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@
| MetricScrapingPeriodSeconds | int | > 0 | 15 | Period of Prometheus metrics scrapping |
| GRPCConnectionTimeoutSeconds | int | > 0 | 60 | Timeout for establishing a gRPC connection |
| GRPCFunctionTimeoutSeconds | int | > 0 | 90 | Maximum time given to function to execute[^4] |
| DAGMode | bool | true/false | false | Sequential invocation of all functions one after another |
| DAGMode | bool | true/false | false | Parallel invocation of DAG with each function acting as entry function |
| DAGTracePath | string | string | data/traces/sampled_150/20 | Folder with Azure trace dimensions used for DAG Invocation. |
[^1]: The second granularity feature interprets each column of the trace as a second, rather than as a minute, and
generates IAT for each second. This feature is useful for fine-grained and precise invocation scheduling in experiments
involving stable low load.
Expand Down
6 changes: 6 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,13 @@ require (
github.com/go-pdf/fpdf v0.9.0 // indirect
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
github.com/pkg/sftp v1.13.4 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/richardlehane/mscfb v1.0.4 // indirect
github.com/richardlehane/msoleps v1.0.3 // indirect
github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53 // indirect
github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05 // indirect
go.opentelemetry.io/otel/exporters/zipkin v1.8.0 // indirect
golang.org/x/crypto v0.23.0 // indirect
golang.org/x/image v0.14.0 // indirect
Expand All @@ -46,6 +51,7 @@ require (
require (
github.com/containerd/containerd v1.6.13 // indirect
github.com/openzipkin/zipkin-go v0.4.0 // indirect
github.com/xuri/excelize/v2 v2.8.1
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.33.0
go.opentelemetry.io/otel v1.11.1 // indirect
go.opentelemetry.io/otel/sdk v1.8.0 // indirect
Expand Down
13 changes: 13 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
Expand All @@ -145,6 +147,11 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/rabbitmq/amqp091-go v1.1.0/go.mod h1:ogQDLSOACsLPsIq0NpbtiifNZi2YOz0VTJ0kHRghqbM=
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM=
github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk=
github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
github.com/richardlehane/msoleps v1.0.3 h1:aznSZzrwYRl3rLKRT3gUk9am7T/mLNSnJINvN0AQoVM=
github.com/richardlehane/msoleps v1.0.3/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sfreiberg/simplessh v0.0.0-20220719182921-185eafd40485 h1:ZMBZ2DKX1sScUSo9ZUwGI7jCMukslPNQNfZaw9vVyfY=
Expand All @@ -167,6 +174,12 @@ github.com/vhive-serverless/vSwarm/utils/tracing/go v0.0.0-20230926064847-68cc9b
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53 h1:Chd9DkqERQQuHpXjR/HSV1jLZA6uaoiwwH3vSuF3IW0=
github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
github.com/xuri/excelize/v2 v2.8.1 h1:pZLMEwK8ep+CLIUWpWmvW8IWE/yxqG0I1xcN6cVMGuQ=
github.com/xuri/excelize/v2 v2.8.1/go.mod h1:oli1E4C3Pa5RXg1TBXn4ENCXDV5JUMlBluUhG7c+CEE=
github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05 h1:qhbILQo1K3mphbwKh1vNm4oGezE1eF9fQWmNiIpSfI4=
github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.33.0 h1:z6rnla1Asjzn0FrhohzIbDi4bxbtc6EMmQ7f5ZPn+pA=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.33.0/go.mod h1:y/SlJpJQPd2UzfBCj0E9Flk9FDCtTyqUmaCB41qFrWI=
Expand Down
8 changes: 8 additions & 0 deletions pkg/common/trace_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@

package common

import "container/list"

type FunctionInvocationStats struct {
HashOwner string
HashApp string
Expand Down Expand Up @@ -98,3 +100,9 @@ type Function struct {

Specification *FunctionSpecification
}

type Node struct {
Function *Function
Branches []*list.List
Depth int
}
1 change: 1 addition & 0 deletions pkg/config/parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ type LoaderConfiguration struct {
GRPCConnectionTimeoutSeconds int `json:"GRPCConnectionTimeoutSeconds"`
GRPCFunctionTimeoutSeconds int `json:"GRPCFunctionTimeoutSeconds"`
DAGMode bool `json:"DAGMode"`
DAGTracePath string `json:"DAGTracePath"`
}

func ReadConfigurationFile(path string) LoaderConfiguration {
Expand Down
Loading

0 comments on commit 486319d

Please sign in to comment.