diff --git a/.gitignore b/.gitignore index a22990c..05fc00e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ vendor/ /.release /.tarballs *.tar.gz +telly.config.* diff --git a/Gopkg.lock b/Gopkg.lock index cc87f39..3209f48 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1,25 +1,6 @@ # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. -[[projects]] - branch = "master" - digest = "1:315c5f2f60c76d89b871c73f9bd5fe689cad96597afd50fb9992228ef80bdd34" - name = "github.com/alecthomas/template" - packages = [ - ".", - "parse", - ] - pruneopts = "UT" - revision = "a0175ee3bccc567396460bf5acd36800cb10c49c" - -[[projects]] - branch = "master" - digest = "1:c198fdc381e898e8fb62b8eb62758195091c313ad18e52a3067366e1dda2fb3c" - name = "github.com/alecthomas/units" - packages = ["."] - pruneopts = "UT" - revision = "2efee857e7cfd4f3d0138cc3cbb1b4966962b93a" - [[projects]] branch = "master" digest = "1:d6afaeed1502aa28e80a4ed0981d570ad91b2579193404256ce672ed0a609e0d" @@ -28,6 +9,14 @@ pruneopts = "UT" revision = "3a771d992973f24aa725d07868b467d1ddfceafb" +[[projects]] + digest = "1:abeb38ade3f32a92943e5be54f55ed6d6e3b6602761d74b4aab4c9dd45c18abd" + name = "github.com/fsnotify/fsnotify" + packages = ["."] + pruneopts = "UT" + revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" + version = "v1.4.7" + [[projects]] branch = "master" digest = "1:36fe9527deed01d2a317617e59304eb2c4ce9f8a24115bcc5c2e37b3aee5bae4" @@ -56,6 +45,25 @@ revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" version = "v1.1.0" +[[projects]] + branch = "master" + digest = "1:a361611b8c8c75a1091f00027767f7779b29cb37c456a71b8f2604c88057ab40" + name = "github.com/hashicorp/hcl" + packages = [ + ".", + "hcl/ast", + "hcl/parser", + "hcl/printer", + "hcl/scanner", + "hcl/strconv", + "hcl/token", + "json/parser", + "json/scanner", + "json/token", + ] + pruneopts = "UT" + revision = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168" + [[projects]] branch = "master" digest = "1:8f57afa9ef1d9205094e9d89b9cb4ecb3123f342c4eb0053d7631181b511e6e4" @@ -80,6 +88,14 @@ revision = "e2ffdb16a802fe2bb95e2e35ff34f0e53aeef34f" version = "v0.1.0" +[[projects]] + digest = "1:c568d7727aa262c32bdf8a3f7db83614f7af0ed661474b24588de635c20024c7" + name = "github.com/magiconair/properties" + packages = ["."] + pruneopts = "UT" + revision = "c2353362d570a7bfa228149c62842019201cfb71" + version = "v1.8.0" + [[projects]] digest = "1:fa610f9fe6a93f4a75e64c83673dfff9bf1a34bbb21e6102021b6bc7850834a3" name = "github.com/mattn/go-isatty" @@ -103,6 +119,14 @@ pruneopts = "UT" revision = "f15292f7a699fcc1a38a80977f80a046874ba8ac" +[[projects]] + digest = "1:95741de3af260a92cc5c7f3f3061e85273f5a81b5db20d4bd68da74bd521675e" + name = "github.com/pelletier/go-toml" + packages = ["."] + pruneopts = "UT" + revision = "c01d1270ff3e442a8a57cddc1c92dc1138598194" + version = "v1.2.0" + [[projects]] digest = "1:d14a5f4bfecf017cb780bdde1b6483e5deb87e12c332544d2c430eda58734bcb" name = "github.com/prometheus/client_golang" @@ -157,19 +181,54 @@ version = "v1.0.6" [[projects]] - digest = "1:c268acaa4a4d94a467980e5e91452eb61c460145765293dc0aed48e5e9919cc6" - name = "github.com/ugorji/go" - packages = ["codec"] + digest = "1:bd1ae00087d17c5a748660b8e89e1043e1e5479d0fea743352cda2f8dd8c4f84" + name = "github.com/spf13/afero" + packages = [ + ".", + "mem", + ] pruneopts = "UT" - revision = "c88ee250d0221a57af388746f5cf03768c21d6e2" + revision = "787d034dfe70e44075ccc060d346146ef53270ad" + version = "v1.1.1" + +[[projects]] + digest = "1:516e71bed754268937f57d4ecb190e01958452336fa73dbac880894164e91c1f" + name = "github.com/spf13/cast" + packages = ["."] + pruneopts = "UT" + revision = "8965335b8c7107321228e3e3702cab9832751bac" + version = "v1.2.0" [[projects]] branch = "master" - digest = "1:7e4543a28ce437be9d263089699c5fd6cefc0f02a63592f7f85c0c4e21245e0a" - name = "github.com/zsais/go-gin-prometheus" + digest = "1:8a020f916b23ff574845789daee6818daf8d25a4852419aae3f0b12378ba432a" + name = "github.com/spf13/jwalterweatherman" + packages = ["."] + pruneopts = "UT" + revision = "14d3d4c518341bea657dd8a226f5121c0ff8c9f2" + +[[projects]] + digest = "1:dab83a1bbc7ad3d7a6ba1a1cc1760f25ac38cdf7d96a5cdd55cd915a4f5ceaf9" + name = "github.com/spf13/pflag" packages = ["."] pruneopts = "UT" - revision = "3f93884fa240fd102425d65ce9781e561ba40496" + revision = "9a97c102cda95a86cec2345a6f09f55a939babf5" + version = "v1.0.2" + +[[projects]] + digest = "1:4fc8a61287ccfb4286e1ca5ad2ce3b0b301d746053bf44ac38cf34e40ae10372" + name = "github.com/spf13/viper" + packages = ["."] + pruneopts = "UT" + revision = "907c19d40d9a6c9bb55f040ff4ae45271a4754b9" + version = "v1.1.0" + +[[projects]] + digest = "1:c268acaa4a4d94a467980e5e91452eb61c460145765293dc0aed48e5e9919cc6" + name = "github.com/ugorji/go" + packages = ["codec"] + pruneopts = "UT" + revision = "c88ee250d0221a57af388746f5cf03768c21d6e2" [[projects]] branch = "master" @@ -207,7 +266,7 @@ revision = "98c5dad5d1a0e8a73845ecc8897d0bd56586511d" [[projects]] - digest = "1:aa4d6967a3237f8367b6bf91503964a77183ecf696f1273e8ad3551bb4412b5f" + digest = "1:4392fcf42d5cf0e3ff78c96b2acf8223d49e4fdc53eb77c99d2f8dfe4680e006" name = "golang.org/x/text" packages = [ "encoding", @@ -222,24 +281,19 @@ "encoding/unicode", "internal/gen", "internal/tag", + "internal/triegen", + "internal/ucd", "internal/utf8internal", "language", "runes", "transform", "unicode/cldr", + "unicode/norm", ] pruneopts = "UT" revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" version = "v0.3.0" -[[projects]] - digest = "1:c06d9e11d955af78ac3bbb26bd02e01d2f61f689e1a3bce2ef6fb683ef8a7f2d" - name = "gopkg.in/alecthomas/kingpin.v2" - packages = ["."] - pruneopts = "UT" - revision = "947dcec5ba9c011838740e680966fd7087a71d0d" - version = "v2.2.6" - [[projects]] digest = "1:1b4724d3c8125f6044925f02b485b74bfec9905cbf579d95aafd1a6c8f8447d3" name = "gopkg.in/go-playground/validator.v8" @@ -264,11 +318,12 @@ "github.com/kr/pretty", "github.com/mitchellh/mapstructure", "github.com/prometheus/client_golang/prometheus", + "github.com/prometheus/client_golang/prometheus/promhttp", "github.com/prometheus/common/version", "github.com/sirupsen/logrus", - "github.com/zsais/go-gin-prometheus", + "github.com/spf13/pflag", + "github.com/spf13/viper", "golang.org/x/net/html/charset", - "gopkg.in/alecthomas/kingpin.v2", ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 31ebde3..546090b 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -49,14 +49,6 @@ name = "github.com/sirupsen/logrus" version = "1.0.6" -[[constraint]] - branch = "master" - name = "github.com/zsais/go-gin-prometheus" - -[[constraint]] - name = "gopkg.in/alecthomas/kingpin.v2" - version = "2.2.6" - [prune] go-tests = true unused-packages = true diff --git a/internal/go-gin-prometheus/middleware.go b/internal/go-gin-prometheus/middleware.go new file mode 100644 index 0000000..f3d7477 --- /dev/null +++ b/internal/go-gin-prometheus/middleware.go @@ -0,0 +1,402 @@ +// Package ginprometheus provides a Logrus logger for Gin requests. Slightly modified to remove spammy logs. +// For more info see https://github.com/zsais/go-gin-prometheus/pull/22. +package ginprometheus + +import ( + "bytes" + "io/ioutil" + "net/http" + "os" + "strconv" + "time" + + "github.com/gin-gonic/gin" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + log "github.com/sirupsen/logrus" +) + +var defaultMetricPath = "/metrics" + +// Standard default metrics +// counter, counter_vec, gauge, gauge_vec, +// histogram, histogram_vec, summary, summary_vec +var reqCnt = &Metric{ + ID: "reqCnt", + Name: "requests_total", + Description: "How many HTTP requests processed, partitioned by status code and HTTP method.", + Type: "counter_vec", + Args: []string{"code", "method", "handler", "host", "url"}} + +var reqDur = &Metric{ + ID: "reqDur", + Name: "request_duration_seconds", + Description: "The HTTP request latencies in seconds.", + Type: "summary"} + +var resSz = &Metric{ + ID: "resSz", + Name: "response_size_bytes", + Description: "The HTTP response sizes in bytes.", + Type: "summary"} + +var reqSz = &Metric{ + ID: "reqSz", + Name: "request_size_bytes", + Description: "The HTTP request sizes in bytes.", + Type: "summary"} + +var standardMetrics = []*Metric{ + reqCnt, + reqDur, + resSz, + reqSz, +} + +/* +RequestCounterURLLabelMappingFn is a function which can be supplied to the middleware to control +the cardinality of the request counter's "url" label, which might be required in some contexts. +For instance, if for a "/customer/:name" route you don't want to generate a time series for every +possible customer name, you could use this function: +func(c *gin.Context) string { + url := c.Request.URL.String() + for _, p := range c.Params { + if p.Key == "name" { + url = strings.Replace(url, p.Value, ":name", 1) + break + } + } + return url +} +which would map "/customer/alice" and "/customer/bob" to their template "/customer/:name". +*/ +type RequestCounterURLLabelMappingFn func(c *gin.Context) string + +// Metric is a definition for the name, description, type, ID, and +// prometheus.Collector type (i.e. CounterVec, Summary, etc) of each metric +type Metric struct { + MetricCollector prometheus.Collector + ID string + Name string + Description string + Type string + Args []string +} + +// Prometheus contains the metrics gathered by the instance and its path +type Prometheus struct { + reqCnt *prometheus.CounterVec + reqDur, reqSz, resSz prometheus.Summary + router *gin.Engine + listenAddress string + Ppg PrometheusPushGateway + + MetricsList []*Metric + MetricsPath string + + ReqCntURLLabelMappingFn RequestCounterURLLabelMappingFn +} + +// PrometheusPushGateway contains the configuration for pushing to a Prometheus pushgateway (optional) +type PrometheusPushGateway struct { + + // Push interval in seconds + PushIntervalSeconds time.Duration + + // Push Gateway URL in format http://domain:port + // where JOBNAME can be any string of your choice + PushGatewayURL string + + // Local metrics URL where metrics are fetched from, this could be ommited in the future + // if implemented using prometheus common/expfmt instead + MetricsURL string + + // pushgateway job name, defaults to "gin" + Job string +} + +// NewPrometheus generates a new set of metrics with a certain subsystem name +func NewPrometheus(subsystem string, customMetricsList ...[]*Metric) *Prometheus { + + var metricsList []*Metric + + if len(customMetricsList) > 1 { + panic("Too many args. NewPrometheus( string, ).") + } else if len(customMetricsList) == 1 { + metricsList = customMetricsList[0] + } + + for _, metric := range standardMetrics { + metricsList = append(metricsList, metric) + } + + p := &Prometheus{ + MetricsList: metricsList, + MetricsPath: defaultMetricPath, + ReqCntURLLabelMappingFn: func(c *gin.Context) string { + return c.Request.URL.String() // i.e. by default do nothing, i.e. return URL as is + }, + } + + p.registerMetrics(subsystem) + + return p +} + +// SetPushGateway sends metrics to a remote pushgateway exposed on pushGatewayURL +// every pushIntervalSeconds. Metrics are fetched from metricsURL +func (p *Prometheus) SetPushGateway(pushGatewayURL, metricsURL string, pushIntervalSeconds time.Duration) { + p.Ppg.PushGatewayURL = pushGatewayURL + p.Ppg.MetricsURL = metricsURL + p.Ppg.PushIntervalSeconds = pushIntervalSeconds + p.startPushTicker() +} + +// SetPushGatewayJob job name, defaults to "gin" +func (p *Prometheus) SetPushGatewayJob(j string) { + p.Ppg.Job = j +} + +// SetListenAddress for exposing metrics on address. If not set, it will be exposed at the +// same address of the gin engine that is being used +func (p *Prometheus) SetListenAddress(address string) { + p.listenAddress = address + if p.listenAddress != "" { + p.router = gin.Default() + } +} + +// SetListenAddressWithRouter for using a separate router to expose metrics. (this keeps things like GET /metrics out of +// your content's access log). +func (p *Prometheus) SetListenAddressWithRouter(listenAddress string, r *gin.Engine) { + p.listenAddress = listenAddress + if len(p.listenAddress) > 0 { + p.router = r + } +} + +func (p *Prometheus) setMetricsPath(e *gin.Engine) { + + if p.listenAddress != "" { + p.router.GET(p.MetricsPath, prometheusHandler()) + p.runServer() + } else { + e.GET(p.MetricsPath, prometheusHandler()) + } +} + +func (p *Prometheus) setMetricsPathWithAuth(e *gin.Engine, accounts gin.Accounts) { + + if p.listenAddress != "" { + p.router.GET(p.MetricsPath, gin.BasicAuth(accounts), prometheusHandler()) + p.runServer() + } else { + e.GET(p.MetricsPath, gin.BasicAuth(accounts), prometheusHandler()) + } + +} + +func (p *Prometheus) runServer() { + if p.listenAddress != "" { + go p.router.Run(p.listenAddress) + } +} + +func (p *Prometheus) getMetrics() []byte { + response, _ := http.Get(p.Ppg.MetricsURL) + + defer response.Body.Close() + body, _ := ioutil.ReadAll(response.Body) + + return body +} + +func (p *Prometheus) getPushGatewayURL() string { + h, _ := os.Hostname() + if p.Ppg.Job == "" { + p.Ppg.Job = "gin" + } + return p.Ppg.PushGatewayURL + "/metrics/job/" + p.Ppg.Job + "/instance/" + h +} + +func (p *Prometheus) sendMetricsToPushGateway(metrics []byte) { + req, err := http.NewRequest("POST", p.getPushGatewayURL(), bytes.NewBuffer(metrics)) + client := &http.Client{} + if _, err = client.Do(req); err != nil { + log.WithError(err).Errorln("Error sending to push gateway") + } +} + +func (p *Prometheus) startPushTicker() { + ticker := time.NewTicker(time.Second * p.Ppg.PushIntervalSeconds) + go func() { + for range ticker.C { + p.sendMetricsToPushGateway(p.getMetrics()) + } + }() +} + +// NewMetric associates prometheus.Collector based on Metric.Type +func NewMetric(m *Metric, subsystem string) prometheus.Collector { + var metric prometheus.Collector + switch m.Type { + case "counter_vec": + metric = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: subsystem, + Name: m.Name, + Help: m.Description, + }, + m.Args, + ) + case "counter": + metric = prometheus.NewCounter( + prometheus.CounterOpts{ + Subsystem: subsystem, + Name: m.Name, + Help: m.Description, + }, + ) + case "gauge_vec": + metric = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: subsystem, + Name: m.Name, + Help: m.Description, + }, + m.Args, + ) + case "gauge": + metric = prometheus.NewGauge( + prometheus.GaugeOpts{ + Subsystem: subsystem, + Name: m.Name, + Help: m.Description, + }, + ) + case "histogram_vec": + metric = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Subsystem: subsystem, + Name: m.Name, + Help: m.Description, + }, + m.Args, + ) + case "histogram": + metric = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Subsystem: subsystem, + Name: m.Name, + Help: m.Description, + }, + ) + case "summary_vec": + metric = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Subsystem: subsystem, + Name: m.Name, + Help: m.Description, + }, + m.Args, + ) + case "summary": + metric = prometheus.NewSummary( + prometheus.SummaryOpts{ + Subsystem: subsystem, + Name: m.Name, + Help: m.Description, + }, + ) + } + return metric +} + +func (p *Prometheus) registerMetrics(subsystem string) { + + for _, metricDef := range p.MetricsList { + metric := NewMetric(metricDef, subsystem) + if err := prometheus.Register(metric); err != nil { + log.WithError(err).Errorf("%s could not be registered in Prometheus", metricDef.Name) + } + switch metricDef { + case reqCnt: + p.reqCnt = metric.(*prometheus.CounterVec) + case reqDur: + p.reqDur = metric.(prometheus.Summary) + case resSz: + p.resSz = metric.(prometheus.Summary) + case reqSz: + p.reqSz = metric.(prometheus.Summary) + } + metricDef.MetricCollector = metric + } +} + +// Use adds the middleware to a gin engine. +func (p *Prometheus) Use(e *gin.Engine) { + e.Use(p.handlerFunc()) + p.setMetricsPath(e) +} + +// UseWithAuth adds the middleware to a gin engine with BasicAuth. +func (p *Prometheus) UseWithAuth(e *gin.Engine, accounts gin.Accounts) { + e.Use(p.handlerFunc()) + p.setMetricsPathWithAuth(e, accounts) +} + +func (p *Prometheus) handlerFunc() gin.HandlerFunc { + return func(c *gin.Context) { + if c.Request.URL.String() == p.MetricsPath { + c.Next() + return + } + + start := time.Now() + reqSz := computeApproximateRequestSize(c.Request) + + c.Next() + + status := strconv.Itoa(c.Writer.Status()) + elapsed := float64(time.Since(start)) / float64(time.Second) + resSz := float64(c.Writer.Size()) + + p.reqDur.Observe(elapsed) + url := p.ReqCntURLLabelMappingFn(c) + p.reqCnt.WithLabelValues(status, c.Request.Method, c.HandlerName(), c.Request.Host, url).Inc() + p.reqSz.Observe(float64(reqSz)) + p.resSz.Observe(resSz) + } +} + +func prometheusHandler() gin.HandlerFunc { + h := promhttp.Handler() + return func(c *gin.Context) { + h.ServeHTTP(c.Writer, c.Request) + } +} + +// From https://github.com/DanielHeckrath/gin-prometheus/blob/master/gin_prometheus.go +func computeApproximateRequestSize(r *http.Request) int { + s := 0 + if r.URL != nil { + s = len(r.URL.String()) + } + + s += len(r.Method) + s += len(r.Proto) + for name, values := range r.Header { + s += len(name) + for _, value := range values { + s += len(value) + } + } + s += len(r.Host) + + // N.B. r.Form and r.MultipartForm are assumed to be included in r.URL. + + if r.ContentLength != -1 { + s += int(r.ContentLength) + } + return s +} diff --git a/m3u/main.go b/internal/m3uplus/main.go similarity index 96% rename from m3u/main.go rename to internal/m3uplus/main.go index 5aa449f..43d2606 100644 --- a/m3u/main.go +++ b/internal/m3uplus/main.go @@ -1,4 +1,5 @@ -package m3u +// Package m3uplus provides a M3U Plus parser. +package m3uplus import ( "bytes" @@ -13,7 +14,7 @@ import ( // Playlist is a type that represents an m3u playlist containing 0 or more tracks type Playlist struct { - Tracks []*Track + Tracks []Track } // Track represents an m3u track @@ -86,7 +87,7 @@ func decodeLine(playlist *Playlist, line string, lineNumber int) error { switch { case strings.HasPrefix(line, "#EXTINF:"): - track := &Track{ + track := Track{ Raw: line, LineNumber: lineNumber, } diff --git a/internal/providers/custom.go b/internal/providers/custom.go new file mode 100644 index 0000000..e6a93c0 --- /dev/null +++ b/internal/providers/custom.go @@ -0,0 +1,89 @@ +package providers + +import ( + "strconv" + "strings" + + m3u "github.com/tombowditch/telly/internal/m3uplus" + "github.com/tombowditch/telly/internal/xmltv" +) + +type customProvider struct { + BaseConfig Configuration +} + +func newCustomProvider(config *Configuration) (Provider, error) { + return &customProvider{*config}, nil +} + +func (i *customProvider) Name() string { + return i.BaseConfig.Name +} + +func (i *customProvider) PlaylistURL() string { + return i.BaseConfig.M3U +} + +func (i *customProvider) EPGURL() string { + return i.BaseConfig.EPG +} + +// ParseTrack matches the provided M3U track an XMLTV channel and returns a ProviderChannel. +func (i *customProvider) ParseTrack(track m3u.Track, channelMap map[string]xmltv.Channel) (*ProviderChannel, error) { + channelVal := track.Tags["tvg-chno"] + if i.BaseConfig.ChannelNumberKey != "" { + channelVal = track.Tags[i.BaseConfig.ChannelNumberKey] + } + + chanNum := 0 + + if channelNumber, channelNumberErr := strconv.Atoi(channelVal); channelNumberErr == nil { + chanNum = channelNumber + } + + nameVal := track.Name + if i.BaseConfig.NameKey != "" { + nameVal = track.Tags[i.BaseConfig.NameKey] + } + + logoVal := track.Tags["tvg-logo"] + if i.BaseConfig.LogoKey != "" { + logoVal = track.Tags[i.BaseConfig.LogoKey] + } + + pChannel := &ProviderChannel{ + Name: nameVal, + Logo: logoVal, + Number: chanNum, + StreamURL: track.URI, + StreamID: chanNum, + HD: strings.Contains(strings.ToLower(track.Name), "hd"), + StreamFormat: "Unknown", + Track: track, + OnDemand: false, + } + + epgVal := track.Tags["tvg-id"] + if i.BaseConfig.EPGMatchKey != "" { + epgVal = track.Tags[i.BaseConfig.EPGMatchKey] + } + + if xmlChan, ok := channelMap[epgVal]; ok { + pChannel.EPGMatch = epgVal + pChannel.EPGChannel = &xmlChan + } + + return pChannel, nil +} + +func (i *customProvider) ProcessProgramme(programme xmltv.Programme) *xmltv.Programme { + return &programme +} + +func (i *customProvider) Configuration() Configuration { + return i.BaseConfig +} + +func (i *customProvider) RegexKey() string { + return i.BaseConfig.FilterKey +} diff --git a/providers/eternal.go b/internal/providers/eternal.go similarity index 64% rename from providers/eternal.go rename to internal/providers/eternal.go index fc59a4f..72bc1b6 100644 --- a/providers/eternal.go +++ b/internal/providers/eternal.go @@ -1,4 +1,4 @@ package providers -// M3U:http://live.eternaltv.net:25461/get.php?username=xxxxxxx&password=xxxxxx&output=ts&type=m3u_plus -// XMLTV: http://live.eternaltv.net:25461/xmltv.php?username=xxxxx&password=xxxxx&type=m3u_plus&output=ts +// M3U:http://live.eternaltv.net:25461/get.php?username=xxxxxxx&password=xxxxxx&output=ts&type=m3uplus +// XMLTV: http://live.eternaltv.net:25461/xmltv.php?username=xxxxx&password=xxxxx&type=m3uplus&output=ts diff --git a/providers/hellraiser.go b/internal/providers/hellraiser.go similarity index 79% rename from providers/hellraiser.go rename to internal/providers/hellraiser.go index de264a8..0608474 100644 --- a/providers/hellraiser.go +++ b/internal/providers/hellraiser.go @@ -1,4 +1,4 @@ package providers -// Playlist URL: http://liquidit.info:8080/get.php?username=xxxx&password=xxxxxxx&type=m3u_plus&output=ts +// Playlist URL: http://liquidit.info:8080/get.php?username=xxxx&password=xxxxxxx&type=m3uplus&output=ts // XMLTV URL: http://liquidit.info:8080/xmltv.php?username=xxxxxx&password=xxxxxx diff --git a/internal/providers/iptv-epg.go b/internal/providers/iptv-epg.go new file mode 100644 index 0000000..63c5602 --- /dev/null +++ b/internal/providers/iptv-epg.go @@ -0,0 +1,92 @@ +package providers + +import ( + "fmt" + "strconv" + "strings" + + m3u "github.com/tombowditch/telly/internal/m3uplus" + "github.com/tombowditch/telly/internal/xmltv" +) + +// M3U: http://iptv-epg.com/.m3u +// XMLTV: http://iptv-epg.com/.xml + +type iptvepg struct { + BaseConfig Configuration +} + +func newIPTVEPG(config *Configuration) (Provider, error) { + return &iptvepg{*config}, nil +} + +func (i *iptvepg) Name() string { + return "IPTV-EPG" +} + +func (i *iptvepg) PlaylistURL() string { + return fmt.Sprintf("http://iptv-epg.com/%s.m3u", i.BaseConfig.Username) +} + +func (i *iptvepg) EPGURL() string { + return fmt.Sprintf("http://iptv-epg.com/%s.xml", i.BaseConfig.Password) +} + +// ParseTrack matches the provided M3U track an XMLTV channel and returns a ProviderChannel. +func (i *iptvepg) ParseTrack(track m3u.Track, channelMap map[string]xmltv.Channel) (*ProviderChannel, error) { + channelVal := track.Tags["tvg-chno"] + if i.BaseConfig.ChannelNumberKey != "" { + channelVal = track.Tags[i.BaseConfig.ChannelNumberKey] + } + + channelNumber, channelNumberErr := strconv.Atoi(channelVal) + if channelNumberErr != nil { + return nil, channelNumberErr + } + + nameVal := track.Name + if i.BaseConfig.NameKey != "" { + nameVal = track.Tags[i.BaseConfig.NameKey] + } + + logoVal := track.Tags["tvg-logo"] + if i.BaseConfig.LogoKey != "" { + logoVal = track.Tags[i.BaseConfig.LogoKey] + } + + pChannel := &ProviderChannel{ + Name: nameVal, + Logo: logoVal, + Number: channelNumber, + StreamURL: track.URI, + StreamID: channelNumber, + HD: strings.Contains(strings.ToLower(track.Name), "hd"), + StreamFormat: "Unknown", + Track: track, + OnDemand: false, + } + + epgVal := track.Tags["tvg-id"] + if i.BaseConfig.EPGMatchKey != "" { + epgVal = track.Tags[i.BaseConfig.EPGMatchKey] + } + + if xmlChan, ok := channelMap[epgVal]; ok { + pChannel.EPGMatch = epgVal + pChannel.EPGChannel = &xmlChan + } + + return pChannel, nil +} + +func (i *iptvepg) ProcessProgramme(programme xmltv.Programme) *xmltv.Programme { + return &programme +} + +func (i *iptvepg) Configuration() Configuration { + return i.BaseConfig +} + +func (i *iptvepg) RegexKey() string { + return "group-title" +} diff --git a/internal/providers/iris.go b/internal/providers/iris.go new file mode 100644 index 0000000..01d10ec --- /dev/null +++ b/internal/providers/iris.go @@ -0,0 +1,4 @@ +package providers + +// http://irislinks.net:83/get.php?username=username&password=password&type=m3uplus&output=ts +// http://irislinks.net:83/xmltv.php?username=username&password=password diff --git a/internal/providers/main.go b/internal/providers/main.go new file mode 100644 index 0000000..f772532 --- /dev/null +++ b/internal/providers/main.go @@ -0,0 +1,97 @@ +package providers + +import ( + "regexp" + "strings" + + m3u "github.com/tombowditch/telly/internal/m3uplus" + "github.com/tombowditch/telly/internal/xmltv" +) + +var streamNumberRegex = regexp.MustCompile(`/(\d+).(ts|.*.m3u8)`).FindAllStringSubmatch +var channelNumberRegex = regexp.MustCompile(`^[0-9]+[[:space:]]?$`).MatchString +var callSignRegex = regexp.MustCompile(`^[A-Z0-9]+$`).MatchString +var hdRegex = regexp.MustCompile(`hd|4k`) + +type Configuration struct { + Name string `json:"-"` + Provider string + + Username string `json:"username"` + Password string `json:"password"` + + M3U string `json:"-"` + EPG string `json:"-"` + + VideoOnDemand bool `json:"-"` + + Filter string + FilterKey string + FilterRaw bool + + SortKey string + SortReverse bool + + Favorites []string + FavoriteTag string + + CacheFiles bool + + NameKey string + LogoKey string + ChannelNumberKey string + EPGMatchKey string +} + +func (i *Configuration) GetProvider() (Provider, error) { + switch strings.ToLower(i.Provider) { + case "vaders": + return newVaders(i) + case "iptv-epg", "iptvepg": + return newIPTVEPG(i) + default: + return newCustomProvider(i) + } +} + +// ProviderChannel describes a channel available in the providers lineup with necessary pieces parsed into fields. +type ProviderChannel struct { + Name string + StreamID int // Should be the integer just before .ts. + Number int + Logo string + StreamURL string + HD bool + Quality string + OnDemand bool + StreamFormat string + Favorite bool + + EPGMatch string + EPGChannel *xmltv.Channel + EPGProgrammes []xmltv.Programme + Track m3u.Track +} + +// Provider describes a IPTV provider configuration. +type Provider interface { + Name() string + PlaylistURL() string + EPGURL() string + + // These are functions to extract information from playlists. + ParseTrack(track m3u.Track, channelMap map[string]xmltv.Channel) (*ProviderChannel, error) + ProcessProgramme(programme xmltv.Programme) *xmltv.Programme + + RegexKey() string + Configuration() Configuration +} + +func contains(s []string, e string) bool { + for _, ss := range s { + if e == ss { + return true + } + } + return false +} diff --git a/providers/tnt.go b/internal/providers/tnt.go similarity index 83% rename from providers/tnt.go rename to internal/providers/tnt.go index 2f8295f..3960706 100644 --- a/providers/tnt.go +++ b/internal/providers/tnt.go @@ -1,4 +1,4 @@ package providers -// M3U: http://thesepeanutz.xyz:2052/get.php?username=xxx&password=xxx&type=m3u_plus&output=ts +// M3U: http://thesepeanutz.xyz:2052/get.php?username=xxx&password=xxx&type=m3uplus&output=ts // XMLTV: http://thesepeanutz.xyz:2052/xmltv.php?username=xxx&password=xxx diff --git a/internal/providers/vaders.go b/internal/providers/vaders.go new file mode 100644 index 0000000..bcb2fa6 --- /dev/null +++ b/internal/providers/vaders.go @@ -0,0 +1,133 @@ +package providers + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" + + log "github.com/sirupsen/logrus" + m3u "github.com/tombowditch/telly/internal/m3uplus" + "github.com/tombowditch/telly/internal/xmltv" +) + +// This regex matches and extracts the following URLs. +// http://vapi.vaders.tv/play/dvr/${start}/123.ts?duration=3600&token= +// http://vapi.vaders.tv/play/123.ts?token= +// http://vapi.vaders.tv/play/vod/123.mp4.m3u8?token= +// http://vapi.vaders.tv/play/vod/123.avi.m3u8?token= +// http://vapi.vaders.tv/play/vod/123.mkv.m3u8?token= +var vadersURL = regexp.MustCompile(`/(vod/|dvr/\${start}/)?(\d+).(ts|.*.m3u8)\?(duration=\d+&)?token=`).FindAllStringSubmatch + +// M3U: http://api.vaders.tv/vget?username=xxx&password=xxx&format=ts +// XMLTV: http://vaders.tv/p2.xml + +type vader struct { + BaseConfig Configuration + + Token string `json:"-"` +} + +func newVaders(config *Configuration) (Provider, error) { + tok, tokErr := json.Marshal(config) + if tokErr != nil { + return nil, tokErr + } + + return &vader{*config, base64.StdEncoding.EncodeToString(tok)}, nil +} + +func (v *vader) Name() string { + return "Vaders.tv" +} + +func (v *vader) PlaylistURL() string { + return fmt.Sprintf("http://api.vaders.tv/vget?username=%s&password=%s&vod=%t&format=ts", v.BaseConfig.Username, v.BaseConfig.Password, v.BaseConfig.VideoOnDemand) +} + +func (v *vader) EPGURL() string { + return "http://vaders.tv/p2.xml.gz" +} + +// ParseTrack matches the provided M3U track an XMLTV channel and returns a ProviderChannel. +func (v *vader) ParseTrack(track m3u.Track, channelMap map[string]xmltv.Channel) (*ProviderChannel, error) { + streamURL := vadersURL(track.URI, -1)[0] + + vod := strings.Contains(streamURL[1], "vod") + + if v.BaseConfig.VideoOnDemand == false && vod { + return nil, nil + } + + channelID, channelIDErr := strconv.Atoi(streamURL[2]) + if channelIDErr != nil { + return nil, channelIDErr + } + + nameVal := track.Tags["tvg-name"] + if v.BaseConfig.NameKey != "" { + nameVal = track.Tags[v.BaseConfig.NameKey] + } + + logoVal := track.Tags["tvg-logo"] + if v.BaseConfig.LogoKey != "" { + logoVal = track.Tags[v.BaseConfig.LogoKey] + } + + pChannel := &ProviderChannel{ + Name: nameVal, + Logo: logoVal, + StreamURL: track.URI, + StreamID: channelID, + HD: strings.Contains(strings.ToLower(track.Tags["tvg-name"]), "hd"), + StreamFormat: streamURL[3], + Track: track, + OnDemand: vod, + } + + if xmlChan, ok := channelMap[track.Tags["tvg-id"]]; ok { + pChannel.EPGMatch = track.Tags["tvg-id"] + pChannel.EPGChannel = &xmlChan + + for _, displayName := range xmlChan.DisplayNames { + if channelNumberRegex(displayName.Value) { + if chanNum, chanNumErr := strconv.Atoi(displayName.Value); chanNumErr == nil { + pChannel.Number = chanNum + } + } + } + } + + favoriteTag := "tvg-id" + + if v.BaseConfig.FavoriteTag != "" { + favoriteTag = v.BaseConfig.FavoriteTag + } + + if _, ok := track.Tags[favoriteTag]; !ok { + log.Panicf("The specified favorite tag (%s) doesn't exist on the track with URL %s", favoriteTag, track.URI) + return nil, nil + } + + pChannel.Favorite = contains(v.BaseConfig.Favorites, track.Tags[favoriteTag]) + + return pChannel, nil +} + +func (v *vader) ProcessProgramme(programme xmltv.Programme) *xmltv.Programme { + for idx, title := range programme.Titles { + programme.Titles[idx].Value = strings.Replace(title.Value, " [New!]", "", -1) + } + + return &programme +} + +func (v *vader) Configuration() Configuration { + return v.BaseConfig +} + +func (v *vader) RegexKey() string { + return "group-title" +} diff --git a/xmltv/xmltv.dtd b/internal/xmltv/xmltv.dtd similarity index 100% rename from xmltv/xmltv.dtd rename to internal/xmltv/xmltv.dtd diff --git a/xmltv/xmltv.go b/internal/xmltv/xmltv.go similarity index 99% rename from xmltv/xmltv.go rename to internal/xmltv/xmltv.go index 911c275..5567f1f 100644 --- a/xmltv/xmltv.go +++ b/internal/xmltv/xmltv.go @@ -65,6 +65,10 @@ type Channel struct { Icons []Icon `xml:"icon,omitempty" json:"icons,omitempty"` URLs []string `xml:"url,omitempty" json:"urls,omitempty" ` ID string `xml:"id,attr" json:"id,omitempty" ` + + // These fields are outside of the XMLTV spec. + // LCN is the local channel number. Plex will show it in place of the channel ID if it exists. + LCN int `xml:"lcn" json:"lcn,omitempty"` } // Programme details of a single programme transmission @@ -102,10 +106,6 @@ type Programme struct { Videoplus string `xml:"videoplus,attr,omitempty" json:"videoplus,omitempty"` Channel string `xml:"channel,attr" json:"channel"` Clumpidx string `xml:"clumpidx,attr,omitempty" json:"clumpidx,omitempty"` - - // These fields are outside of the XMLTV spec. - // LCN is the local channel number. Plex will show it in place of the channel ID if it exists. - LCN int `xml:"lcn,attr" json:"lcn,omitempty"` } // CommonElement element structure that is common, i.e. Italy diff --git a/xmltv/xmltv_test.go b/internal/xmltv/xmltv_test.go similarity index 94% rename from xmltv/xmltv_test.go rename to internal/xmltv/xmltv_test.go index 0cdc479..f2eec7c 100644 --- a/xmltv/xmltv_test.go +++ b/internal/xmltv/xmltv_test.go @@ -17,7 +17,7 @@ func dummyReader(charset string, input io.Reader) (io.Reader, error) { } func TestDecode(t *testing.T) { - // Example downloaded from http://wiki.xmltv.org/index.php/XMLTVFormat + // Example downloaded from http://wiki.xmltv.org/index.php/internal/xmltvFormat // One may check it with `xmllint --noout --dtdvalid xmltv.dtd example.xml` f, err := os.Open("example.xml") if err != nil { @@ -60,7 +60,7 @@ func TestDecode(t *testing.T) { }, Icons: []Icon{ Icon{ - Source: `file://C:\Perl\site/share/xmltv/icons/KERA.gif`, + Source: `file://C:\Perl\site/share/internal/xmltv/icons/KERA.gif`, }, }, } diff --git a/lineup.go b/lineup.go index 4b3badb..3881f7c 100644 --- a/lineup.go +++ b/lineup.go @@ -1,6 +1,7 @@ package main import ( + "compress/gzip" "encoding/xml" "fmt" "io" @@ -8,360 +9,303 @@ import ( "os" "regexp" "sort" - "strconv" "strings" - "time" "github.com/spf13/viper" - "github.com/tombowditch/telly/m3u" - "github.com/tombowditch/telly/providers" - "github.com/tombowditch/telly/xmltv" + m3u "github.com/tombowditch/telly/internal/m3uplus" + "github.com/tombowditch/telly/internal/providers" + "github.com/tombowditch/telly/internal/xmltv" ) -var channelNumberRegex = regexp.MustCompile(`^[0-9]+[[:space:]]?$`).MatchString -var callSignRegex = regexp.MustCompile(`^[A-Z0-9]+$`).MatchString -var hdRegex = regexp.MustCompile(`hd|4k`) - -// Track describes a single M3U segment. This struct includes m3u.Track as well as specific IPTV fields we want to get. -type Track struct { - *m3u.Track - SafeURI string `json:"URI"` - Catchup string `m3u:"catchup" json:",omitempty"` - CatchupDays string `m3u:"catchup-days" json:",omitempty"` - CatchupSource string `m3u:"catchup-source" json:",omitempty"` - GroupTitle string `m3u:"group-title" json:",omitempty"` - TvgID string `m3u:"tvg-id" json:",omitempty"` - TvgLogo string `m3u:"tvg-logo" json:",omitempty"` - TvgName string `m3u:"tvg-name" json:",omitempty"` - TvgChannelNumber string `m3u:"tvg-chno" json:",omitempty"` - ChannelID string `m3u:"channel-id" json:",omitempty"` - - XMLTVChannel *xmlTVChannel `json:",omitempty"` - XMLTVProgrammes *[]xmltv.Programme `json:",omitempty"` +// var channelNumberRegex = regexp.MustCompile(`^[0-9]+[[:space:]]?$`).MatchString +// var callSignRegex = regexp.MustCompile(`^[A-Z0-9]+$`).MatchString +// var hdRegex = regexp.MustCompile(`hd|4k`) + +// hdHomeRunLineupItem is a HDHomeRun specification compatible representation of a Track available in the lineup. +type hdHomeRunLineupItem struct { + XMLName xml.Name `xml:"Program" json:"-"` + + AudioCodec string `xml:",omitempty" json:",omitempty"` + DRM convertibleBoolean `xml:",omitempty" json:",string,omitempty"` + Favorite convertibleBoolean `xml:",omitempty" json:",string,omitempty"` + GuideName string `xml:",omitempty" json:",omitempty"` + GuideNumber int `xml:",omitempty" json:",string,omitempty"` + HD convertibleBoolean `xml:",omitempty" json:",string,omitempty"` + URL string `xml:",omitempty" json:",omitempty"` + VideoCodec string `xml:",omitempty" json:",omitempty"` + + provider providers.Provider + providerChannel providers.ProviderChannel } -func (t *Track) PrettyName() string { - if t.XMLTVChannel != nil { - return t.XMLTVChannel.LongName - } else if t.TvgName != "" { - return t.TvgName - } else if t.Track.Name != "" { - return t.Track.Name +func newHDHRItem(provider *providers.Provider, providerChannel *providers.ProviderChannel) hdHomeRunLineupItem { + return hdHomeRunLineupItem{ + DRM: convertibleBoolean(false), + GuideName: providerChannel.Name, + GuideNumber: providerChannel.Number, + Favorite: convertibleBoolean(providerChannel.Favorite), + HD: convertibleBoolean(providerChannel.HD), + URL: fmt.Sprintf("http://%s/auto/v%d", viper.GetString("web.base-address"), providerChannel.Number), + provider: *provider, + providerChannel: *providerChannel, } - - return t.Name -} - -// Playlist describes a single M3U playlist. -type Playlist struct { - *m3u.Playlist - *M3UFile - - Tracks []Track - Channels []HDHomeRunChannel - TracksCount int - FilteredTracksCount int - EPGProvided bool -} - -// Filter will filter the raw m3u.Playlist m3u.Track slice into the Track slice of the Playlist. -func (p *Playlist) Filter() error { - for _, oldTrack := range p.Playlist.Tracks { - track := Track{ - Track: oldTrack, - SafeURI: safeStringsRegex.ReplaceAllStringFunc(oldTrack.URI, stringSafer), - } - - if unmarshalErr := oldTrack.UnmarshalTags(&track); unmarshalErr != nil { - return unmarshalErr - } - - if GetStringAsRegex("filter.regexstr").MatchString(track.Raw) == viper.GetBool("filter.regexinclusive") { - p.Tracks = append(p.Tracks, track) - } - } - - return nil -} - -// M3UFile describes a path and transport to a M3U provided in the configuration. -type M3UFile struct { - Path string `json:"-"` - SafePath string `json:"Path"` - Transport string } -// HDHomeRunChannel is a HDHomeRun specification compatible representation of a Track available in the Lineup. -type HDHomeRunChannel struct { - AudioCodec string `json:",omitempty"` - DRM convertibleBoolean `json:",string,omitempty"` - Favorite convertibleBoolean `json:",string,omitempty"` - GuideName string `json:",omitempty"` - GuideNumber int `json:",string,omitempty"` - HD convertibleBoolean `json:",string,omitempty"` - URL string `json:",omitempty"` - VideoCodec string `json:",omitempty"` - - track *Track -} - -// Lineup is a collection of tracks -type Lineup struct { - Providers []providers.Provider - - Playlists []Playlist - PlaylistsCount int - TracksCount int - FilteredTracksCount int - - StartingChannelNumber int - channelNumber int +// lineup contains the state of the application. +type lineup struct { + Sources []providers.Provider - Refreshing bool - LastRefreshed time.Time `json:",omitempty"` + Scanning bool - xmlTvChannelMap map[string]xmlTVChannel - channelsInXMLTv []string - xmlTv xmltv.TV - xmlTvSourceInfoURL []string - xmlTvSourceInfoName []string - xmlTvSourceDataURL []string + // Stores the channel number for found channels without a number. + assignedChannelNumber int + // If true, use channel numbers found in EPG, if any, before assigning. xmlTVChannelNumbers bool - chanNumToURLMap map[string]string + channels map[int]hdHomeRunLineupItem } -// NewLineup returns a new Lineup for the given config struct. -func NewLineup() *Lineup { - tv := xmltv.TV{ - GeneratorInfoName: namespaceWithVersion, - GeneratorInfoURL: "https://github.com/tombowditch/telly", - } - - lineup := &Lineup{ - xmlTVChannelNumbers: viper.GetBool("iptv.xmltv-channels"), - chanNumToURLMap: make(map[string]string), - xmlTv: tv, - xmlTvChannelMap: make(map[string]xmlTVChannel), - StartingChannelNumber: viper.GetInt("iptv.starting-channel"), - channelNumber: viper.GetInt("iptv.starting-channel"), - Refreshing: false, - LastRefreshed: time.Now(), - } - +// newLineup returns a new lineup for the given config struct. +func newLineup() *lineup { var cfgs []providers.Configuration if unmarshalErr := viper.UnmarshalKey("source", &cfgs); unmarshalErr != nil { log.WithError(unmarshalErr).Panicln("Unable to unmarshal source configuration to slice of providers.Configuration, check your configuration!") } + if viper.IsSet("iptv.playlist") { + log.Warnln("Legacy --iptv.playlist argument or environment variable provided, using Custom provider with default configuration, this may fail! If so, you should use a configuration file for full flexibility.") + regexStr := ".*" + if viper.IsSet("filter.regex") { + regexStr = viper.GetString("filter.regex") + } + cfgs = append(cfgs, providers.Configuration{ + Name: "Legacy provider created using arguments/environment variables", + M3U: viper.GetString("iptv.playlist"), + Provider: "custom", + Filter: regexStr, + FilterRaw: true, + }) + } + + lineup := &lineup{ + assignedChannelNumber: viper.GetInt("iptv.starting-channel"), + xmlTVChannelNumbers: viper.GetBool("iptv.xmltv-channels"), + channels: make(map[int]hdHomeRunLineupItem), + } + for _, cfg := range cfgs { - log.Infoln("Adding provider", cfg.Name) provider, providerErr := cfg.GetProvider() if providerErr != nil { panic(providerErr) } - if addErr := lineup.AddProvider(provider); addErr != nil { - log.WithError(addErr).Panicln("error adding new provider to lineup") - } + + lineup.Sources = append(lineup.Sources, provider) } return lineup } -// AddProvider adds a new Provider to the Lineup. -func (l *Lineup) AddProvider(provider providers.Provider) error { - reader, info, readErr := l.getM3U(provider.PlaylistURL()) - if readErr != nil { - log.WithError(readErr).Errorln("error getting m3u") - return readErr - } - - rawPlaylist, err := m3u.Decode(reader) - if err != nil { - log.WithError(err).Errorln("unable to parse m3u file") - return err - } +// Scan processes all sources. +func (l *lineup) Scan() error { - if provider.EPGURL() != "" { - epg, epgReadErr := l.getXMLTV(provider.EPGURL()) - if epgReadErr != nil { - log.WithError(epgReadErr).Errorln("error getting XMLTV") - return epgReadErr - } + l.Scanning = true - chanMap, chanMapErr := l.processXMLTV(epg) - if chanMapErr != nil { - log.WithError(chanMapErr).Errorln("Error building channel mapping") - } + totalAddedChannels := 0 - for chanID, chann := range chanMap { - l.xmlTvChannelMap[chanID] = chann + for _, provider := range l.Sources { + addedChannels, providerErr := l.processProvider(provider) + if providerErr != nil { + log.WithError(providerErr).Errorln("error when processing provider") } + totalAddedChannels = totalAddedChannels + addedChannels } - playlist, playlistErr := l.NewPlaylist(provider, rawPlaylist, info) - if playlistErr != nil { - return playlistErr + if totalAddedChannels > 420 { + log.Panicf("telly has loaded more than 420 channels (%d) into the lineup. Plex does not deal well with more than this amount and will more than likely hang when trying to fetch channels. You must use regular expressions to filter out channels. You can also start another Telly instance.", totalAddedChannels) } - l.Playlists = append(l.Playlists, playlist) - l.PlaylistsCount = len(l.Playlists) - l.TracksCount = l.TracksCount + playlist.TracksCount - l.FilteredTracksCount = l.FilteredTracksCount + playlist.FilteredTracksCount + l.Scanning = false return nil } -// NewPlaylist will return a new and filtered Playlist for the given m3u.Playlist and M3UFile. -func (l *Lineup) NewPlaylist(provider providers.Provider, rawPlaylist *m3u.Playlist, info *M3UFile) (Playlist, error) { - hasEPG := provider.EPGURL() != "" - playlist := Playlist{rawPlaylist, info, nil, nil, len(rawPlaylist.Tracks), 0, hasEPG} +func (l *lineup) processProvider(provider providers.Provider) (int, error) { + addedChannels := 0 + m3u, channelMap, programmeMap, prepareErr := l.prepareProvider(provider) + if prepareErr != nil { + log.WithError(prepareErr).Errorln("error when preparing provider") + } - if filterErr := playlist.Filter(); filterErr != nil { - log.WithError(filterErr).Errorln("error during filtering of channels, check your regex and try again") - return playlist, filterErr + if provider.Configuration().SortKey != "" { + sortKey := provider.Configuration().SortKey + sort.Slice(m3u.Tracks, func(i, j int) bool { + if _, ok := m3u.Tracks[i].Tags[sortKey]; ok { + log.Panicf("the provided sort key (%s) doesn't exist in the M3U!", sortKey) + return false + } + ii := m3u.Tracks[i].Tags[sortKey] + jj := m3u.Tracks[j].Tags[sortKey] + if provider.Configuration().SortReverse { + return ii < jj + } + return ii > jj + }) } - for idx, track := range playlist.Tracks { - tt, channelNumber, hd, ttErr := l.processTrack(provider, track) - if ttErr != nil { - return playlist, ttErr + for _, track := range m3u.Tracks { + // First, we run the filter. + if !l.FilterTrack(provider, track) { + log.Debugf("Channel %s didn't pass the provider (%s) filter, skipping!", track.Name, provider.Name()) + return addedChannels, nil } - if hasEPG && tt.XMLTVChannel == nil { - log.Warnf("%s (#%d) is not being exposed to Plex because there was no EPG data found.", tt.Name, channelNumber) - continue + // Then we do the provider specific translation to a hdHomeRunLineupItem. + channel, channelErr := provider.ParseTrack(track, channelMap) + if channelErr != nil { + return addedChannels, channelErr } - playlist.Tracks[idx] = *tt - - guideName := tt.PrettyName() + channel, processErr := l.processProviderChannel(channel, programmeMap) + if processErr != nil { + log.WithError(processErr).Errorln("error processing track") + } else if channel == nil { + log.Infof("Channel %s was returned empty from the provider (%s)", track.Name, provider.Name()) + continue + } + addedChannels = addedChannels + 1 - log.Debugln("Assigning", channelNumber, l.channelNumber, "to", guideName) + l.channels[channel.Number] = newHDHRItem(&provider, channel) + } - hdhr := HDHomeRunChannel{ - GuideNumber: channelNumber, - GuideName: guideName, - URL: fmt.Sprintf("http://%s/auto/v%d", viper.GetString("web.base-address"), channelNumber), - HD: convertibleBoolean(hd), - DRM: convertibleBoolean(false), - } + log.Infof("Loaded %d channels into the lineup from %s", addedChannels, provider.Name()) - if !channelExists(playlist.Channels, hdhr) { - playlist.Channels = append(playlist.Channels, hdhr) - l.chanNumToURLMap[strconv.Itoa(channelNumber)] = tt.Track.URI - } + return addedChannels, nil +} - if channelNumber == l.channelNumber { // Only increment lineup channel number if its for a channel that didnt have a XMLTV entry. - l.channelNumber = l.channelNumber + 1 - } +func (l *lineup) prepareProvider(provider providers.Provider) (*m3u.Playlist, map[string]xmltv.Channel, map[string][]xmltv.Programme, error) { + cacheFiles := provider.Configuration().CacheFiles + reader, m3uErr := getM3U(provider.PlaylistURL(), cacheFiles) + if m3uErr != nil { + log.WithError(m3uErr).Errorln("unable to get m3u file") + return nil, nil, nil, m3uErr } - sort.Slice(l.xmlTv.Channels, func(i, j int) bool { - first, _ := strconv.Atoi(l.xmlTv.Channels[i].ID) - second, _ := strconv.Atoi(l.xmlTv.Channels[j].ID) - return first < second - }) + rawPlaylist, err := m3u.Decode(reader) + if err != nil { + log.WithError(err).Errorln("unable to parse m3u file") + return nil, nil, nil, err + } - playlist.FilteredTracksCount = len(playlist.Tracks) - exposedChannels.Add(float64(playlist.FilteredTracksCount)) - log.Debugf("Added %d channels to the lineup", playlist.FilteredTracksCount) + channelMap, programmeMap, epgErr := l.prepareEPG(provider, cacheFiles) + if epgErr != nil { + log.WithError(epgErr).Errorln("error when parsing EPG") + return nil, nil, nil, epgErr + } - return playlist, nil + return rawPlaylist, channelMap, programmeMap, nil } -func (l Lineup) processTrack(provider providers.Provider, track Track) (*Track, int, bool, error) { +func (l *lineup) processProviderChannel(channel *providers.ProviderChannel, programmeMap map[string][]xmltv.Programme) (*providers.ProviderChannel, error) { + if channel.EPGChannel != nil { + channel.EPGProgrammes = programmeMap[channel.EPGMatch] + } - hd := hdRegex.MatchString(strings.ToLower(track.Track.Raw)) - channelNumber := l.channelNumber + if !l.xmlTVChannelNumbers || channel.Number == 0 { + channel.Number = l.assignedChannelNumber + l.assignedChannelNumber = l.assignedChannelNumber + 1 + } - if xmlChan, ok := l.xmlTvChannelMap[track.TvgID]; ok { - log.Debugln("found an entry in xmlTvChannelMap for", track.Name) - if l.xmlTVChannelNumbers && xmlChan.Number != 0 { - channelNumber = xmlChan.Number - } else { - xmlChan.Number = channelNumber - } - l.channelsInXMLTv = append(l.channelsInXMLTv, track.TvgID) - track.XMLTVChannel = &xmlChan - l.xmlTv.Channels = append(l.xmlTv.Channels, xmlChan.RemappedChannel(track)) - if xmlChan.Programmes != nil { - track.XMLTVProgrammes = &xmlChan.Programmes - for _, programme := range xmlChan.Programmes { - newProgramme := programme - for idx, title := range programme.Titles { - programme.Titles[idx].Value = strings.Replace(title.Value, " [New!]", "", -1) // Hardcoded fix for Vaders - } - newProgramme.Channel = strconv.Itoa(channelNumber) - if hd { - if newProgramme.Video == nil { - newProgramme.Video = &xmltv.Video{} - } - newProgramme.Video.Quality = "HDTV" - } - l.xmlTv.Programmes = append(l.xmlTv.Programmes, newProgramme) - } - } + if channel.EPGChannel != nil && channel.EPGChannel.LCN == 0 { + channel.EPGChannel.LCN = channel.Number } - return &track, channelNumber, hd, nil + if channel.Logo != "" && channel.EPGChannel != nil && !containsIcon(channel.EPGChannel.Icons, channel.Logo) { + channel.EPGChannel.Icons = append(channel.EPGChannel.Icons, xmltv.Icon{Source: channel.Logo}) + } + + return channel, nil } -// Refresh will rescan all playlists for any channel changes. -func (l Lineup) Refresh() error { +func (l *lineup) FilterTrack(provider providers.Provider, track m3u.Track) bool { + config := provider.Configuration() + if config.Filter == "" { + return true + } - if l.Refreshing { - log.Warnln("A refresh is already underway yet, another one was requested") - return nil + filterRegex, regexErr := regexp.Compile(config.Filter) + if regexErr != nil { + log.WithError(regexErr).Panicln("your regex is invalid") + return false } - log.Warnln("Refreshing the lineup!") + if config.FilterRaw { + return filterRegex.MatchString(track.Raw) + } - l.Refreshing = true + log.Debugf("track.Tags %+v", track.Tags) - existingPlaylists := make([]Playlist, len(l.Playlists)) - copy(existingPlaylists, l.Playlists) + filterKey := provider.RegexKey() + if config.FilterKey != "" { + if key, ok := track.Tags[config.FilterKey]; key != "" && ok { + filterKey = config.FilterKey + } else { + log.Panicf("the provided filter key (%s) does not exist or is blank", config.FilterKey) + } + } - l.Playlists = nil - l.TracksCount = 0 - l.FilteredTracksCount = 0 - l.StartingChannelNumber = 0 + if _, ok := track.Tags[filterKey]; !ok { + log.Panicf("Provided filter key %s doesn't exist in M3U tags", filterKey) + } - // FIXME: Re-implement AddProvider to use a provider. - // for _, playlist := range existingPlaylists { - // if addErr := l.AddProvider(playlist.M3UFile.Path); addErr != nil { - // return addErr - // } - // } + log.Debugf("Checking if filter (%s) matches string %s", config.Filter, track.Tags[filterKey]) - log.Infoln("Done refreshing the lineup!") + return filterRegex.MatchString(track.Tags[filterKey]) - l.LastRefreshed = time.Now() - l.Refreshing = false +} - return nil +func (l *lineup) prepareEPG(provider providers.Provider, cacheFiles bool) (map[string]xmltv.Channel, map[string][]xmltv.Programme, error) { + var epg *xmltv.TV + epgChannelMap := make(map[string]xmltv.Channel) + epgProgrammeMap := make(map[string][]xmltv.Programme) + if provider.EPGURL() != "" { + var epgErr error + epg, epgErr = getXMLTV(provider.EPGURL(), cacheFiles) + if epgErr != nil { + return epgChannelMap, epgProgrammeMap, epgErr + } + + for _, channel := range epg.Channels { + epgChannelMap[channel.ID] = channel + + for _, programme := range epg.Programmes { + if programme.Channel == channel.ID { + epgProgrammeMap[channel.ID] = append(epgProgrammeMap[channel.ID], *provider.ProcessProgramme(programme)) + } + } + } + } + + return epgChannelMap, epgProgrammeMap, nil } -func (l *Lineup) getM3U(path string) (io.Reader, *M3UFile, error) { +func getM3U(path string, cacheFiles bool) (io.Reader, error) { safePath := safeStringsRegex.ReplaceAllStringFunc(path, stringSafer) log.Infof("Loading M3U from %s", safePath) - file, transport, err := l.getFile(path) + file, _, err := getFile(path, cacheFiles) if err != nil { - return nil, nil, err + return nil, err } - return file, &M3UFile{ - Path: path, - SafePath: safePath, - Transport: transport, - }, nil + return file, nil } -func (l *Lineup) getXMLTV(path string) (*xmltv.TV, error) { - file, _, err := l.getFile(path) +func getXMLTV(path string, cacheFiles bool) (*xmltv.TV, error) { + safePath := safeStringsRegex.ReplaceAllStringFunc(path, stringSafer) + log.Infof("Loading XMLTV from %s", safePath) + file, _, err := getFile(path, cacheFiles) if err != nil { return nil, err } @@ -376,118 +320,69 @@ func (l *Lineup) getXMLTV(path string) (*xmltv.TV, error) { return tvSetup, nil } -func (l *Lineup) getFile(path string) (io.Reader, string, error) { - safePath := safeStringsRegex.ReplaceAllStringFunc(path, stringSafer) - log.Infof("Loading file from %s", safePath) - +func getFile(path string, cacheFiles bool) (io.Reader, string, error) { transport := "disk" if strings.HasPrefix(strings.ToLower(path), "http") { + resp, err := http.Get(path) if err != nil { return nil, transport, err } - //defer resp.Body.Close() - return resp.Body, transport, nil - } - - file, fileErr := os.Open(path) - if fileErr != nil { - return nil, transport, fileErr - } - - return file, transport, nil -} - -type xmlTVChannel struct { - ID string - Number int - CallSign string - ShortName string - LongName string + // defer func() { + // err := resp.Body.Close() + // if err != nil { + // log.WithError(err).Panicln("error when closing HTTP body reader") + // } + // }() + + if strings.HasSuffix(strings.ToLower(path), ".gz") { + log.Infof("File (%s) is gzipp'ed, ungzipping now, this might take a while", path) + gz, gzErr := gzip.NewReader(resp.Body) + if gzErr != nil { + return nil, transport, gzErr + } - NumberAssigned bool + defer func() { + err := gz.Close() + if err != nil { + log.WithError(err).Panicln("error when closing gzip reader") + } + }() - Programmes []xmltv.Programme + if cacheFiles { + return writeFile(path, transport, gz) + } - Original xmltv.Channel -} + return gz, transport, nil + } -func (x *xmlTVChannel) RemappedChannel(t Track) xmltv.Channel { - newX := x.Original - newX.ID = strconv.Itoa(x.Number) - if t.TvgLogo != "" { - newX.Icons = append(newX.Icons, xmltv.Icon{Source: t.TvgLogo}) - } - if t.Track.Name != "" { - newX.DisplayNames = append(newX.DisplayNames, xmltv.CommonElement{Value: t.Track.Name}) - } - return newX -} + if cacheFiles { + return writeFile(path, transport, resp.Body) + } -func (l *Lineup) processXMLTV(tv *xmltv.TV) (map[string]xmlTVChannel, error) { - programmeMap := make(map[string][]xmltv.Programme) - for _, programme := range tv.Programmes { - programmeMap[programme.Channel] = append(programmeMap[programme.Channel], programme) + return resp.Body, transport, nil } - channelMap := make(map[string]xmlTVChannel, 0) - for _, tvChann := range tv.Channels { - xTVChan := &xmlTVChannel{ - ID: tvChann.ID, - Original: tvChann, - } - if programmes, ok := programmeMap[tvChann.ID]; ok { - xTVChan.Programmes = programmes - } - if channelNumberRegex(tvChann.ID) { - xTVChan.Number, _ = strconv.Atoi(tvChann.ID) - } - displayNames := []string{} - for _, displayName := range tvChann.DisplayNames { - displayNames = append(displayNames, displayName.Value) - } - sort.StringSlice(displayNames).Sort() - for i := 0; i < 10; i++ { - extractDisplayNames(displayNames, xTVChan) - } - channelMap[xTVChan.ID] = *xTVChan - // Duplicate this to first display-name just in case the M3U and XMLTV differ significantly. - for _, dn := range tvChann.DisplayNames { - channelMap[dn.Value] = *xTVChan - } + file, fileErr := os.Open(path) + if fileErr != nil { + return nil, transport, fileErr } - return channelMap, nil + return file, transport, nil } -func extractDisplayNames(displayNames []string, xTVChan *xmlTVChannel) { - for _, displayName := range displayNames { - if channelNumberRegex(displayName) { - if chanNum, chanNumErr := strconv.Atoi(displayName); chanNumErr == nil { - log.Debugln(displayName, "is channel number!") - xTVChan.Number = chanNum - } - } else if !strings.HasPrefix(displayName, fmt.Sprintf("%d", xTVChan.Number)) { - if xTVChan.LongName == "" { - xTVChan.LongName = displayName - log.Debugln(displayName, "is long name!") - } else if !callSignRegex(displayName) && len(xTVChan.LongName) < len(displayName) { - xTVChan.ShortName = xTVChan.LongName - xTVChan.LongName = displayName - log.Debugln(displayName, "is NEW long name, replacing", xTVChan.ShortName) - } else if callSignRegex(displayName) { - xTVChan.CallSign = displayName - log.Debugln(displayName, "is call sign!") - } - } - } +func writeFile(path, transport string, reader io.Reader) (io.Reader, string, error) { + // buf := new(bytes.Buffer) + // buf.ReadFrom(reader) + // buf.Bytes() + return reader, transport, nil } -func channelExists(s []HDHomeRunChannel, e HDHomeRunChannel) bool { - for _, a := range s { - if a.GuideName == e.GuideName { +func containsIcon(s []xmltv.Icon, e string) bool { + for _, ss := range s { + if e == ss.Source { return true } } diff --git a/main.go b/main.go index 7746d66..66fb61f 100644 --- a/main.go +++ b/main.go @@ -27,7 +27,6 @@ var ( Hooks: make(logrus.LevelHooks), Level: logrus.DebugLevel, } - opts = config{} exposedChannels = prometheus.NewGauge( prometheus.GaugeOpts{ @@ -71,8 +70,8 @@ func main() { flag.String("filter.regex", ".*", "Use regex to filter for channels that you want. A basic example would be .*UK.*. $(TELLY_FILTER_REGEX)") // Web flags - flag.String("web.listen-address", "localhost:6077", "Address to listen on for web interface and telemetry $(TELLY_WEB_LISTEN_ADDRESS)") - flag.String("web.base-address", "localhost:6077", "The address to expose via discovery. Useful with reverse proxy $(TELLY_WEB_BASE_ADDRESS)") + flag.StringP("web.listen-address", "l", "localhost:6077", "Address to listen on for web interface and telemetry $(TELLY_WEB_LISTEN_ADDRESS)") + flag.StringP("web.base-address", "b", "localhost:6077", "The address to expose via discovery. Useful with reverse proxy $(TELLY_WEB_BASE_ADDRESS)") // Log flags flag.String("log.level", logrus.InfoLevel.String(), "Only log messages with the given severity or above. Valid levels: [debug, info, warn, error, fatal] $(TELLY_LOG_LEVEL)") @@ -84,25 +83,63 @@ func main() { flag.Int("iptv.starting-channel", 10000, "The channel number to start exposing from. $(TELLY_IPTV_STARTING_CHANNEL)") flag.Bool("iptv.xmltv-channels", true, "Use channel numbers discovered via XMLTV file, if provided. $(TELLY_IPTV_XMLTV_CHANNELS)") + // Misc flags + flag.StringP("config.file", "c", "", "Path to your config file. If not set, configuration is searched for in the current working directory, $HOME/.telly/ and /etc/telly/. If provided, it will override all other arguments and environment variables. $(TELLY_CONFIG_FILE)") + flag.Bool("version", false, "Show application version") + flag.CommandLine.AddGoFlagSet(fflag.CommandLine) + + deprecatedFlags := []string{ + "discovery.device-id", + "discovery.device-friendly-name", + "discovery.device-auth", + "discovery.device-manufacturer", + "discovery.device-model-number", + "discovery.device-firmware-name", + "discovery.device-firmware-version", + "discovery.ssdp", + "iptv.playlist", + "iptv.streams", + "iptv.starting-channel", + "iptv.xmltv-channels", + "filter.regex-inclusive", + "filter.regex", + } + + for _, depFlag := range deprecatedFlags { + if depErr := flag.CommandLine.MarkDeprecated(depFlag, "use the configuration file instead."); depErr != nil { + log.WithError(depErr).Panicf("error marking flag %s as deprecated", depFlag) + } + } + flag.Parse() - viper.BindPFlags(flag.CommandLine) - viper.SetConfigName("telly.config") // name of config file (without extension) - viper.AddConfigPath("/etc/telly/") // path to look for the config file in - viper.AddConfigPath("$HOME/.telly") // call multiple times to add many search paths - viper.AddConfigPath(".") // optionally look for config in the working directory - viper.SetEnvPrefix(namespace) - viper.AutomaticEnv() - err := viper.ReadInConfig() // Find and read the config file - if err != nil { // Handle errors reading the config file + if bindErr := viper.BindPFlags(flag.CommandLine); bindErr != nil { + log.WithError(bindErr).Panicln("error binding flags to viper") + } + + if flag.Lookup("version").Changed { + fmt.Println(version.Print(namespace)) + os.Exit(0) + } + + if flag.Lookup("config.file").Changed { + viper.SetConfigFile(flag.Lookup("config.file").Value.String()) + } else { + viper.SetConfigName("telly.config") + viper.AddConfigPath("/etc/telly/") + viper.AddConfigPath("$HOME/.telly") + viper.AddConfigPath(".") + viper.SetEnvPrefix(namespace) + viper.AutomaticEnv() + } + + err := viper.ReadInConfig() + if err != nil { if _, ok := err.(viper.ConfigFileNotFoundError); !ok { log.WithError(err).Panicln("fatal error while reading config file:") } } - log.Infoln("Starting telly", version.Info()) - log.Infoln("Build context", version.BuildContext()) - prometheus.MustRegister(version.NewCollector("telly"), exposedChannels) level, parseLevelErr := logrus.ParseLevel(viper.GetString("log.level")) @@ -111,11 +148,32 @@ func main() { } log.SetLevel(level) + log.Infoln("telly is preparing to go live", version.Info()) + log.Debugln("Build context", version.BuildContext()) + + validateConfig() + + viper.Set("discovery.device-friendly-name", fmt.Sprintf("HDHomerun (%s)", viper.GetString("discovery.device-friendly-name"))) + viper.Set("discovery.device-uuid", fmt.Sprintf("%d-AE2A-4E54-BBC9-33AF7D5D6A92", viper.GetInt("discovery.device-id"))) + if log.Level == logrus.DebugLevel { - js, _ := json.MarshalIndent(viper.AllSettings(), "", " ") + js, jsErr := json.MarshalIndent(viper.AllSettings(), "", " ") + if jsErr != nil { + log.WithError(jsErr).Panicln("error marshal indenting viper config to JSON") + } log.Debugf("Loaded configuration %s", js) } + lineup := newLineup() + + if scanErr := lineup.Scan(); scanErr != nil { + log.WithError(scanErr).Panicln("Error scanning lineup!") + } + + serve(lineup) +} + +func validateConfig() { if viper.IsSet("filter.regexstr") { if _, regexErr := regexp.Compile(viper.GetString("filter.regex")); regexErr != nil { log.WithError(regexErr).Panicln("Error when compiling regex, is it valid?") @@ -127,33 +185,17 @@ func main() { log.WithError(addrErr).Panic("Error when parsing Listen address, please check the address and try again.") return } + if _, addrErr = net.ResolveTCPAddr("tcp", viper.GetString("web.base-address")); addrErr != nil { log.WithError(addrErr).Panic("Error when parsing Base addresses, please check the address and try again.") return } - if GetTCPAddr("web.base-address").IP.IsUnspecified() { + if getTCPAddr("web.base-address").IP.IsUnspecified() { log.Panicln("base URL is set to 0.0.0.0, this will not work. please use the --web.baseaddress option and set it to the (local) ip address telly is running on.") } - if GetTCPAddr("web.listenaddress").IP.IsUnspecified() && GetTCPAddr("web.base-address").IP.IsLoopback() { + if getTCPAddr("web.listenaddress").IP.IsUnspecified() && getTCPAddr("web.base-address").IP.IsLoopback() { log.Warnln("You are listening on all interfaces but your base URL is localhost (meaning Plex will try and load localhost to access your streams) - is this intended?") } - - viper.Set("discovery.device-friendly-name", fmt.Sprintf("HDHomerun (%s)", viper.GetString("discovery.device-friendly-name"))) - viper.Set("discovery.device-uuid", fmt.Sprintf("%d-AE2A-4E54-BBC9-33AF7D5D6A92", viper.GetInt("discovery.device-id"))) - - if flag.Lookup("iptv.playlist").Changed { - viper.Set("playlists.default.m3u", flag.Lookup("iptv.playlist").Value.String()) - } - - lineup := NewLineup() - - log.Infof("Loaded %d channels into the lineup", lineup.FilteredTracksCount) - - if lineup.FilteredTracksCount > 420 { - log.Panicf("telly has loaded more than 420 channels (%d) into the lineup. Plex does not deal well with more than this amount and will more than likely hang when trying to fetch channels. You must use regular expressions to filter out channels. You can also start another Telly instance.", lineup.FilteredTracksCount) - } - - serve(lineup) } diff --git a/providers/iptv-epg.go b/providers/iptv-epg.go deleted file mode 100644 index d1af649..0000000 --- a/providers/iptv-epg.go +++ /dev/null @@ -1,4 +0,0 @@ -package providers - -// M3U: http://iptv-epg.com/.m3u -// XMLTV: http://iptv-epg.com/.xml diff --git a/providers/main.go b/providers/main.go deleted file mode 100644 index ad7bbe3..0000000 --- a/providers/main.go +++ /dev/null @@ -1,97 +0,0 @@ -package providers - -import ( - "fmt" - "regexp" - "strings" - - log "github.com/sirupsen/logrus" - "github.com/tombowditch/telly/m3u" -) - -var channelNumberExtractor = regexp.MustCompile(`/(\d+).(ts|.*.m3u8)`).FindAllStringSubmatch - -type Configuration struct { - Name string `json:"-"` - Provider string - - Username string `json:"username"` - Password string `json:"password"` - - M3U string `json:"-"` - EPG string `json:"-"` - - VideoOnDemand bool `json:"-"` -} - -func (i *Configuration) GetProvider() (Provider, error) { - switch strings.ToLower(i.Provider) { - case "vaders": - log.Infoln("Source is vaders!") - return newVaders(i) - case "custom": - default: - log.Infoln("source is either custom or unknown, assuming custom!") - } - return nil, nil -} - -// ProviderChannel describes a channel available in the providers lineup with necessary pieces parsed into fields. -type ProviderChannel struct { - Name string - InternalID int // Should be the integer just before .ts. - Number *int - Logo string - StreamURL string - HD bool - Quality string - OnDemand bool - StreamFormat string -} - -// Provider describes a IPTV provider configuration. -type Provider interface { - Name() string - PlaylistURL() string - EPGURL() string - - // These are functions to extract information from playlists. - ParseLine(line m3u.Track) (*ProviderChannel, error) - - AuthenticatedStreamURL(channel *ProviderChannel) string - - MatchPlaylistKey() string -} - -// UnmarshalProviders takes V, a slice of Configuration and transforms it into a slice of Provider. -func UnmarshalProviders(v interface{}) ([]Provider, error) { - providers := make([]Provider, 0) - - uncasted, ok := v.([]interface{}) - if !ok { - panic(fmt.Errorf("provided slice is not of type []Configuration, it is of type %T", v)) - } - - for _, uncastedProvider := range uncasted { - ipProvider := uncastedProvider.(Configuration) - log.Infof("ipProvider %+v", ipProvider) - } - - return providers, nil -} - -// func testProvider() { -// v, vErr := NewVadersTV("hunter1", "hunter2", false) -// if vErr != nil { -// log.WithError(vErr).Errorf("Error setting up %s", v.Name()) -// } -// log.Infoln("Provider name is", v.Name()) -// log.Infoln("Playlist URL is", v.PlaylistURL()) -// log.Infoln("EPG URL is", v.EPGURL()) -// log.Infof("Stream URL is %+v", v.AuthenticatedStreamURL(&ProviderChannel{ -// Name: "Test channel", -// InternalID: 2862, -// })) - -// return -// } diff --git a/providers/vaders.go b/providers/vaders.go deleted file mode 100644 index a324c39..0000000 --- a/providers/vaders.go +++ /dev/null @@ -1,72 +0,0 @@ -package providers - -import ( - "encoding/base64" - "encoding/json" - "fmt" - "strconv" - "strings" - - "github.com/tombowditch/telly/m3u" -) - -// M3U: http://api.vaders.tv/vget?username=xxx&password=xxx&format=ts -// XMLTV: http://vaders.tv/p2.xml - -type vader struct { - provider Configuration - - Token string `json:"-"` -} - -func newVaders(config *Configuration) (Provider, error) { - tok, tokErr := json.Marshal(config) - if tokErr != nil { - return nil, tokErr - } - - return &vader{*config, base64.StdEncoding.EncodeToString(tok)}, nil -} - -func (v *vader) Name() string { - return "Vaders.tv" -} - -func (v *vader) PlaylistURL() string { - return fmt.Sprintf("http://api.vaders.tv/vget?username=%s&password=%s&vod=%t&format=ts", v.provider.Username, v.provider.Password, v.provider.VideoOnDemand) -} - -func (v *vader) EPGURL() string { - return "http://vaders.tv/p2.xml" -} - -func (v *vader) ParseLine(line m3u.Track) (*ProviderChannel, error) { - streamURL := channelNumberExtractor(line.URI, -1)[0] - channelID, channelIDErr := strconv.Atoi(streamURL[1]) - if channelIDErr != nil { - return nil, channelIDErr - } - - // http://vapi.vaders.tv/play/dvr/${start}/TSID.ts?duration=3600&token= - // http://vapi.vaders.tv/play/TSID.ts?token= - // http://vapi.vaders.tv/play/vod/VODID.mp4.m3u8?token= - // http://vapi.vaders.tv/play/vod/VODID.avi.m3u8?token= - // http://vapi.vaders.tv/play/vod/VODID.mkv.m3u8?token= - - return &ProviderChannel{ - Name: line.Tags["tvg-name"], - Logo: line.Tags["tvg-logo"], - StreamURL: line.URI, - InternalID: channelID, - HD: strings.Contains(strings.ToLower(line.Tags["tvg-name"]), "hd"), - StreamFormat: streamURL[2], - }, nil -} - -func (v *vader) AuthenticatedStreamURL(channel *ProviderChannel) string { - return fmt.Sprintf("http://vapi.vaders.tv/play/%d.ts?token=%s", channel.InternalID, v.Token) -} - -func (v *vader) MatchPlaylistKey() string { - return "tvg-id" -} diff --git a/routes.go b/routes.go index 26a2012..774b499 100644 --- a/routes.go +++ b/routes.go @@ -5,24 +5,29 @@ import ( "fmt" "net/http" "sort" + "strconv" + "strings" "time" "github.com/gin-gonic/gin" ssdp "github.com/koron/go-ssdp" "github.com/sirupsen/logrus" "github.com/spf13/viper" - ginprometheus "github.com/zsais/go-gin-prometheus" + ginprometheus "github.com/tombowditch/telly/internal/go-gin-prometheus" + "github.com/tombowditch/telly/internal/xmltv" ) -func serve(lineup *Lineup) { - discoveryData := GetDiscoveryData() +func serve(lineup *lineup) { + discoveryData := getDiscoveryData() log.Debugln("creating device xml") upnp := discoveryData.UPNP() log.Debugln("creating webserver routes") - gin.SetMode(gin.ReleaseMode) + if viper.GetString("log.level") != logrus.DebugLevel.String() { + gin.SetMode(gin.ReleaseMode) + } router := gin.New() router.Use(gin.Recovery()) @@ -43,7 +48,7 @@ func serve(lineup *Lineup) { Source: "Cable", SourceList: []string{"Cable"}, } - if lineup.Refreshing { + if lineup.Scanning { payload = LineupStatus{ ScanInProgress: convertibleBoolean(true), // Gotta fake out Plex. @@ -57,7 +62,7 @@ func serve(lineup *Lineup) { router.POST("/lineup.post", func(c *gin.Context) { scanAction := c.Query("scan") if scanAction == "start" { - if refreshErr := lineup.Refresh(); refreshErr != nil { + if refreshErr := lineup.Scan(); refreshErr != nil { c.AbortWithError(http.StatusInternalServerError, refreshErr) } c.AbortWithStatus(http.StatusOK) @@ -70,6 +75,7 @@ func serve(lineup *Lineup) { }) router.GET("/device.xml", deviceXML(upnp)) router.GET("/lineup.json", serveLineup(lineup)) + router.GET("/lineup.xml", serveLineup(lineup)) router.GET("/auto/:channelID", stream(lineup)) router.GET("/epg.xml", xmlTV(lineup)) router.GET("/debug.json", func(c *gin.Context) { @@ -82,7 +88,9 @@ func serve(lineup *Lineup) { } } - log.Infof("Listening and serving HTTP on %s", viper.GetString("web.listen-address")) + log.Infof("telly is live and on the air!") + log.Infof("Broadcasting on %s", viper.GetString("web.listen-address")) + log.Infof("EPG URL: http://%s/epg.xml", viper.GetString("web.listen-address")) if err := router.Run(viper.GetString("web.listen-address")); err != nil { log.WithError(err).Panicln("Error starting up web server") } @@ -100,40 +108,71 @@ func discovery(data DiscoveryData) gin.HandlerFunc { } } -func lineupStatus(status LineupStatus) gin.HandlerFunc { - return func(c *gin.Context) { - c.JSON(http.StatusOK, status) - } +type hdhrLineupContainer struct { + XMLName xml.Name `xml:"Lineup" json:"-"` + Programs []hdHomeRunLineupItem } -func serveLineup(lineup *Lineup) gin.HandlerFunc { +func serveLineup(lineup *lineup) gin.HandlerFunc { return func(c *gin.Context) { - allChannels := make([]HDHomeRunChannel, 0) - for _, playlist := range lineup.Playlists { - allChannels = append(allChannels, playlist.Channels...) + channels := make([]hdHomeRunLineupItem, 0) + for _, channel := range lineup.channels { + channels = append(channels, channel) + } + sort.Slice(channels, func(i, j int) bool { + return channels[i].GuideNumber < channels[j].GuideNumber + }) + if strings.HasSuffix(c.Request.URL.String(), ".xml") { + buf, marshallErr := xml.MarshalIndent(hdhrLineupContainer{Programs: channels}, "", "\t") + if marshallErr != nil { + c.AbortWithError(http.StatusInternalServerError, fmt.Errorf("error marshalling lineup to XML")) + } + c.Data(http.StatusOK, "application/xml", []byte(``+"\n"+string(buf))) + return } - sort.Slice(allChannels, func(i, j int) bool { return allChannels[i].GuideNumber < allChannels[j].GuideNumber }) - c.JSON(http.StatusOK, allChannels) + c.JSON(http.StatusOK, channels) } } -func xmlTV(lineup *Lineup) gin.HandlerFunc { +func xmlTV(lineup *lineup) gin.HandlerFunc { + epg := &xmltv.TV{ + GeneratorInfoName: namespaceWithVersion, + GeneratorInfoURL: "https://github.com/tombowditch/telly", + } + + for _, channel := range lineup.channels { + if channel.providerChannel.EPGChannel != nil { + epg.Channels = append(epg.Channels, *channel.providerChannel.EPGChannel) + epg.Programmes = append(epg.Programmes, channel.providerChannel.EPGProgrammes...) + } + } + + sort.Slice(epg.Channels, func(i, j int) bool { return epg.Channels[i].LCN < epg.Channels[j].LCN }) + return func(c *gin.Context) { - buf, _ := xml.MarshalIndent(lineup.xmlTv, "", "\t") + buf, marshallErr := xml.MarshalIndent(epg, "", "\t") + if marshallErr != nil { + c.AbortWithError(http.StatusInternalServerError, fmt.Errorf("error marshalling EPG to XML")) + } c.Data(http.StatusOK, "application/xml", []byte(xml.Header+``+"\n"+string(buf))) } } -func stream(lineup *Lineup) gin.HandlerFunc { +func stream(lineup *lineup) gin.HandlerFunc { return func(c *gin.Context) { - channelID := c.Param("channelID")[1:] + channelIDStr := c.Param("channelID")[1:] + channelID, channelIDErr := strconv.Atoi(channelIDStr) + if channelIDErr != nil { + c.AbortWithError(http.StatusBadRequest, fmt.Errorf("that (%s) doesn't appear to be a valid channel number", channelIDStr)) + return + } - if url, ok := lineup.chanNumToURLMap[channelID]; ok { - log.Infof("Serving channel number %s", channelID) - c.Redirect(http.StatusMovedPermanently, url) + if channel, ok := lineup.channels[channelID]; ok { + log.Infof("Serving channel number %d", channelID) + c.Redirect(http.StatusMovedPermanently, channel.providerChannel.Track.URI) return } - c.AbortWithError(http.StatusNotFound, fmt.Errorf("unknown channel number %s", channelID)) + c.AbortWithError(http.StatusNotFound, fmt.Errorf("unknown channel number %d", channelID)) } } diff --git a/structs.go b/structs.go index 40d169b..c3977a3 100644 --- a/structs.go +++ b/structs.go @@ -4,51 +4,8 @@ import ( "encoding/json" "encoding/xml" "fmt" - "net" - "regexp" ) -type config struct { - Filter struct { - RegexInclusive bool `toml:"Filter.RegexInclusive"` - Regex *regexp.Regexp `toml:"-"` - RegexStr string `toml:"Filter.Regex"` - } - - IPTV struct { - Playlists []string `toml:"IPTV.Playlists"` - ConcurrentStreams int `toml:"IPTV.ConcurrentStreams"` - StartingChannel int `toml:"IPTV.StartingChannel"` - XMLTVChannelNumbers bool `toml:"IPTV.XMLTVChannelNumbers"` - } - - Discovery struct { - DeviceAuth string `toml:"Discovery.DeviceAuth"` - DeviceID int `toml:"Discovery.DeviceID"` - DeviceUUID string `toml:"Discovery.DeviceUUID"` - FriendlyName string `toml:"Discovery.FriendlyName"` - Manufacturer string `toml:"Discovery.Manufacturer"` - ModelNumber string `toml:"Discovery.ModelNumber"` - FirmwareName string `toml:"Discovery.FirmwareName"` - FirmwareVersion string `toml:"Discovery.FirmwareVersion"` - SSDP bool `toml:"Discovery.SSDP"` - } - - Log struct { - LogRequests bool `toml:"Log.Requests"` - Level string `toml:"Log.Level"` - } - - Web struct { - ListenAddress *net.TCPAddr `toml:"-"` - BaseAddress *net.TCPAddr `toml:"-"` - ListenAddressStr string `toml:"Web.ListenAddress"` - BaseAddressStr string `toml:"Web.BaseAddress"` - } - - lineup *Lineup -} - // DiscoveryData contains data about telly to expose in the HDHomeRun format for Plex detection. type DiscoveryData struct { FriendlyName string @@ -136,3 +93,29 @@ func (bit *convertibleBoolean) UnmarshalJSON(data []byte) error { } return nil } + +// MarshalXML used to determine if the element is present or not. see https://stackoverflow.com/a/46516243 +func (bit *convertibleBoolean) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + var bitSetVar int8 + if *bit { + bitSetVar = 1 + } + + return e.EncodeElement(bitSetVar, start) +} + +// UnmarshalXML used to determine if the element is present or not. see https://stackoverflow.com/a/46516243 +func (bit *convertibleBoolean) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + var asString string + if decodeErr := d.DecodeElement(&asString, &start); decodeErr != nil { + return decodeErr + } + if asString == "1" || asString == "true" { + *bit = true + } else if asString == "0" || asString == "false" { + *bit = false + } else { + return fmt.Errorf("Boolean unmarshal error: invalid input %s", asString) + } + return nil +} diff --git a/utils.go b/utils.go index a106b96..2bddd71 100644 --- a/utils.go +++ b/utils.go @@ -3,28 +3,26 @@ package main import ( "fmt" "net" - "regexp" "strconv" "github.com/spf13/viper" ) -func GetTCPAddr(key string) *net.TCPAddr { - addr, _ := net.ResolveTCPAddr("tcp", viper.GetString(key)) +func getTCPAddr(key string) *net.TCPAddr { + addr, addrErr := net.ResolveTCPAddr("tcp", viper.GetString(key)) + if addrErr != nil { + panic(fmt.Errorf("error parsing address %s: %s", viper.GetString(key), addrErr)) + } return addr } -func GetStringAsRegex(key string) *regexp.Regexp { - return regexp.MustCompile(viper.GetString(key)) -} - -func GetDiscoveryData() DiscoveryData { +func getDiscoveryData() DiscoveryData { return DiscoveryData{ FriendlyName: viper.GetString("discovery.device-friendly-name"), Manufacturer: viper.GetString("discovery.device-manufacturer"), ModelNumber: viper.GetString("discovery.device-model-number"), FirmwareName: viper.GetString("discovery.device-firmware-name"), - TunerCount: viper.GetInt("iptv.concurrent-streams"), + TunerCount: viper.GetInt("iptv.streams"), FirmwareVersion: viper.GetString("discovery.device-firmware-version"), DeviceID: strconv.Itoa(viper.GetInt("discovery.device-id")), DeviceAuth: viper.GetString("discovery.device-auth"),