From e1e49426392d77882a49abbcd0d3291e2eb4b13b Mon Sep 17 00:00:00 2001 From: Hylke Visser Date: Wed, 8 Mar 2017 13:20:38 +0100 Subject: [PATCH 01/13] Add AS923 Channel plans Resolves #489 Resolves #491 Additionally, this commit adds a "cache" of Frequency plans, so that we can just copy it instead of regenerating on every uplink. This commit also prevents downlink and activations on illegal uplink frequencies. --- api/protocol/lorawan/lorawan.pb.go | 188 +++++++++++++++-------------- api/protocol/lorawan/lorawan.proto | 9 ++ core/band/band.go | 90 +++++++++++--- core/band/band_test.go | 18 +++ 4 files changed, 200 insertions(+), 105 deletions(-) diff --git a/api/protocol/lorawan/lorawan.pb.go b/api/protocol/lorawan/lorawan.pb.go index 56978c478..02e29f3c2 100644 --- a/api/protocol/lorawan/lorawan.pb.go +++ b/api/protocol/lorawan/lorawan.pb.go @@ -76,18 +76,22 @@ const ( Region_AU_915_928 Region = 4 Region_CN_470_510 Region = 5 Region_AS_923 Region = 6 + Region_AS_920_923 Region = 61 + Region_AS_923_925 Region = 62 Region_KR_920_923 Region = 7 ) var Region_name = map[int32]string{ - 0: "EU_863_870", - 1: "US_902_928", - 2: "CN_779_787", - 3: "EU_433", - 4: "AU_915_928", - 5: "CN_470_510", - 6: "AS_923", - 7: "KR_920_923", + 0: "EU_863_870", + 1: "US_902_928", + 2: "CN_779_787", + 3: "EU_433", + 4: "AU_915_928", + 5: "CN_470_510", + 6: "AS_923", + 61: "AS_920_923", + 62: "AS_923_925", + 7: "KR_920_923", } var Region_value = map[string]int32{ "EU_863_870": 0, @@ -97,6 +101,8 @@ var Region_value = map[string]int32{ "AU_915_928": 4, "CN_470_510": 5, "AS_923": 6, + "AS_920_923": 61, + "AS_923_925": 62, "KR_920_923": 7, } @@ -3835,87 +3841,87 @@ func init() { } var fileDescriptorLorawan = []byte{ - // 1300 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xcc, 0x57, 0xcb, 0x6e, 0xdb, 0xc6, - 0x1a, 0x36, 0x2d, 0x52, 0x97, 0x5f, 0x96, 0xcd, 0x4c, 0x12, 0x1c, 0x9d, 0x24, 0xb0, 0x0d, 0xe1, - 0x1c, 0x1c, 0xc3, 0x38, 0xb5, 0x64, 0x29, 0x89, 0xad, 0x16, 0x28, 0xa0, 0x9b, 0x1b, 0x27, 0xb6, - 0xe4, 0x8c, 0x2c, 0xa4, 0xe8, 0x66, 0x40, 0x93, 0x43, 0x99, 0x96, 0x78, 0xc9, 0x68, 0x64, 0x4b, - 0x5d, 0x75, 0xd1, 0x67, 0xe8, 0x4b, 0x74, 0xdb, 0x45, 0x1f, 0x21, 0xcb, 0x6c, 0xda, 0x45, 0x16, - 0x46, 0x91, 0x27, 0x29, 0x66, 0x48, 0x59, 0xb2, 0xdc, 0xa6, 0x88, 0xd3, 0x45, 0x57, 0xfc, 0xaf, - 0xdf, 0xfc, 0x33, 0xff, 0x4d, 0x82, 0x6a, 0xd7, 0xe1, 0xa7, 0xc3, 0x93, 0x2d, 0xd3, 0x77, 0xf3, - 0xc7, 0xa7, 0xf4, 0xf8, 0xd4, 0xf1, 0xba, 0x83, 0x26, 0xe5, 0x17, 0x3e, 0xeb, 0xe5, 0x39, 0xf7, - 0xf2, 0x46, 0xe0, 0xe4, 0x03, 0xe6, 0x73, 0xdf, 0xf4, 0xfb, 0xf9, 0xbe, 0xcf, 0x8c, 0x0b, 0xc3, - 0x9b, 0x7c, 0xb7, 0xa4, 0x02, 0x25, 0x22, 0xf6, 0xc1, 0x67, 0x33, 0x60, 0x5d, 0xbf, 0xeb, 0x87, - 0x8e, 0x27, 0x43, 0x5b, 0x72, 0x92, 0x91, 0x54, 0xe8, 0x97, 0xfb, 0x55, 0x81, 0xe4, 0x21, 0xe5, - 0x86, 0x65, 0x70, 0x03, 0x95, 0x00, 0x5c, 0xdf, 0x1a, 0xf6, 0x0d, 0xee, 0xf8, 0x5e, 0x36, 0xbd, - 0xae, 0x6c, 0x2c, 0x17, 0xef, 0x6e, 0x4d, 0x0e, 0x3a, 0xbc, 0x52, 0xe1, 0x19, 0x33, 0xf4, 0x10, - 0x52, 0xc2, 0x99, 0x30, 0x83, 0xd3, 0xec, 0xd2, 0xba, 0xb2, 0x91, 0xc2, 0x49, 0x21, 0xc0, 0x06, - 0xa7, 0xe8, 0xdf, 0x90, 0x3c, 0x71, 0x78, 0xa8, 0xcb, 0xac, 0x2b, 0x1b, 0x19, 0x9c, 0x38, 0x71, - 0xb8, 0x54, 0xad, 0x41, 0xda, 0xf4, 0x2d, 0xc7, 0xeb, 0x86, 0xda, 0x65, 0xe9, 0x09, 0xa1, 0x48, - 0x1a, 0xdc, 0x05, 0xcd, 0x26, 0xa6, 0xc7, 0xb3, 0x2b, 0xd2, 0x51, 0xb5, 0x6b, 0x1e, 0x47, 0xff, - 0x83, 0x38, 0xa3, 0x5d, 0x11, 0x9e, 0x2e, 0xc3, 0x5b, 0xb9, 0x0a, 0x0f, 0x4b, 0x31, 0x8e, 0xd4, - 0xb9, 0x9f, 0x14, 0x58, 0x39, 0x1e, 0xd5, 0x7c, 0xcf, 0x76, 0xba, 0x43, 0x16, 0x86, 0xfa, 0xcf, - 0xbf, 0x5f, 0xee, 0x7b, 0x15, 0x50, 0xc5, 0xe4, 0xce, 0xb9, 0x3c, 0xfc, 0x2a, 0x33, 0x4d, 0x48, - 0x18, 0x41, 0x40, 0xe8, 0xd0, 0xc9, 0x2a, 0xeb, 0xca, 0xc6, 0x52, 0xf5, 0xc9, 0xbb, 0xcb, 0xb5, - 0xed, 0xbf, 0xaa, 0x1b, 0xd3, 0x67, 0x34, 0xcf, 0xc7, 0x01, 0x1d, 0x6c, 0x55, 0x82, 0xa0, 0xd1, - 0xd9, 0xc7, 0x71, 0x23, 0x08, 0x1a, 0x43, 0x47, 0xe0, 0x59, 0xf4, 0x5c, 0xe2, 0x2d, 0xde, 0x0a, - 0xaf, 0x4e, 0xcf, 0x25, 0x9e, 0x45, 0xcf, 0x05, 0xde, 0x4b, 0x48, 0x0a, 0x3c, 0xc3, 0xb2, 0x58, - 0x36, 0x26, 0x01, 0x9f, 0xbe, 0xbb, 0x5c, 0x2b, 0x7e, 0x1c, 0x60, 0xc5, 0xb2, 0x18, 0x16, 0x71, - 0x09, 0x02, 0x61, 0x48, 0x79, 0x17, 0x3d, 0x32, 0x20, 0x3d, 0x3a, 0xce, 0xaa, 0xb7, 0xc2, 0x6c, - 0x5e, 0xf4, 0xda, 0x2f, 0xe8, 0x18, 0x27, 0xbc, 0x90, 0x40, 0x39, 0xc8, 0xb0, 0xd1, 0x36, 0xb1, - 0x18, 0xf1, 0x6d, 0x7b, 0x40, 0xb9, 0xac, 0x81, 0x0c, 0x4e, 0xb3, 0xd1, 0x76, 0x9d, 0xb5, 0xa4, - 0x08, 0xdd, 0x87, 0x38, 0x1b, 0x15, 0x89, 0xc5, 0x64, 0xb2, 0x33, 0x58, 0x63, 0xa3, 0x62, 0x9d, - 0x89, 0x4c, 0xb3, 0x11, 0xb1, 0x68, 0xdf, 0x18, 0x4f, 0x32, 0xcd, 0x46, 0x75, 0xc1, 0xa2, 0x0d, - 0x48, 0x98, 0x36, 0xe9, 0x3b, 0x03, 0x2e, 0xb3, 0x9c, 0x9e, 0x29, 0xca, 0xda, 0xde, 0x81, 0x33, - 0xe0, 0x38, 0x6e, 0xda, 0xe2, 0x3b, 0x53, 0xbd, 0x2b, 0x1f, 0xae, 0xde, 0x1f, 0x17, 0x21, 0x71, - 0x48, 0x07, 0x03, 0xa3, 0x4b, 0xd1, 0xff, 0x41, 0x73, 0xc9, 0xa9, 0xc5, 0x64, 0xe6, 0xd3, 0xc5, - 0xcc, 0xb4, 0x60, 0x9f, 0xd5, 0x71, 0x35, 0xf9, 0xe6, 0x72, 0x6d, 0xe1, 0xed, 0xe5, 0x9a, 0x82, - 0x55, 0xf7, 0x99, 0xc5, 0x90, 0x0e, 0x31, 0xd7, 0x31, 0xc3, 0xac, 0x62, 0x41, 0xa2, 0xa7, 0x90, - 0x76, 0x0d, 0x93, 0x04, 0xc6, 0xb8, 0xef, 0x1b, 0x96, 0x4c, 0x4f, 0x7a, 0xb6, 0xec, 0x2b, 0xb5, - 0xa3, 0x50, 0xf5, 0x6c, 0x01, 0x83, 0x6b, 0x98, 0x11, 0x87, 0x5a, 0x70, 0xef, 0xcc, 0x77, 0x3c, - 0xc2, 0xe8, 0xeb, 0x21, 0x1d, 0xf0, 0x2b, 0x00, 0x55, 0x02, 0x3c, 0xbc, 0x02, 0x78, 0xee, 0x3b, - 0x1e, 0x0e, 0x6d, 0xa6, 0x40, 0xe8, 0xec, 0x86, 0x14, 0x1d, 0xc0, 0x5d, 0x09, 0x68, 0x98, 0x26, - 0x0d, 0xa6, 0x78, 0x9a, 0xc4, 0x7b, 0x70, 0x0d, 0xaf, 0x22, 0x4d, 0xa6, 0x70, 0x77, 0xce, 0xe6, - 0x85, 0xd5, 0x14, 0x24, 0x22, 0x32, 0xd7, 0x06, 0x55, 0xbc, 0x05, 0xfa, 0x2f, 0xc4, 0x5d, 0x22, - 0x52, 0x2f, 0x9f, 0x6a, 0xb9, 0xb8, 0x3c, 0xbd, 0xe4, 0xf1, 0x38, 0xa0, 0x58, 0x73, 0xc5, 0x07, - 0xfd, 0x07, 0x34, 0xd7, 0x38, 0xf3, 0x99, 0x7c, 0xa4, 0x6b, 0x56, 0x42, 0x8a, 0x43, 0x65, 0x8e, - 0x01, 0x4c, 0x9f, 0x46, 0x24, 0xc1, 0xfe, 0xc3, 0x24, 0xec, 0xcd, 0x25, 0xc1, 0x16, 0x49, 0xb8, - 0x0f, 0x71, 0x9b, 0x04, 0x3e, 0xe3, 0xf2, 0x08, 0x0d, 0x6b, 0xf6, 0x91, 0xcf, 0xb8, 0x18, 0x09, - 0x36, 0x73, 0xaf, 0x65, 0x62, 0x09, 0x83, 0xcd, 0xdc, 0xc9, 0x45, 0x7e, 0x51, 0x40, 0x15, 0x80, - 0xa8, 0x33, 0xd3, 0x4f, 0x61, 0xc3, 0x7f, 0x2e, 0x8e, 0xf8, 0xd4, 0x9e, 0xca, 0x8b, 0xb8, 0x4c, - 0xce, 0xfa, 0x32, 0xae, 0xf4, 0xcc, 0xd5, 0xf7, 0x6a, 0x9c, 0xf5, 0x67, 0xee, 0xa1, 0xd9, 0x42, - 0x30, 0x9d, 0x51, 0xb1, 0x99, 0x19, 0x5c, 0x10, 0x28, 0x7e, 0xc0, 0x07, 0x59, 0x75, 0x3d, 0x36, - 0x5f, 0x4b, 0x35, 0xdf, 0x75, 0x0d, 0xcf, 0xaa, 0xaa, 0x02, 0x0a, 0x6b, 0x76, 0x2b, 0xe0, 0x83, - 0xdc, 0x29, 0x68, 0xf2, 0x00, 0x51, 0x9d, 0x46, 0x74, 0xa5, 0x24, 0x16, 0x24, 0x5a, 0x85, 0xb4, - 0x61, 0x31, 0x62, 0x98, 0x3d, 0x51, 0x68, 0x32, 0xae, 0x24, 0x4e, 0x19, 0x16, 0xab, 0x98, 0x3d, - 0x4c, 0x5f, 0x4b, 0x0f, 0xb3, 0x27, 0xcf, 0x17, 0x1e, 0x66, 0x4f, 0x0c, 0x64, 0x9b, 0x04, 0xd4, - 0x13, 0x83, 0x54, 0x16, 0x63, 0x12, 0x27, 0xed, 0xa3, 0x90, 0xcf, 0xed, 0xca, 0xac, 0x45, 0x41, - 0x08, 0x67, 0xd3, 0xb1, 0xe4, 0x71, 0x19, 0x2c, 0x48, 0x94, 0x85, 0xc4, 0xe4, 0xf9, 0xc3, 0x16, - 0x99, 0xb0, 0xb9, 0x1f, 0x16, 0x01, 0xdd, 0x2c, 0x65, 0x84, 0xe7, 0x27, 0x6f, 0x39, 0x4a, 0xc4, - 0x27, 0x4c, 0x5f, 0x3c, 0x3f, 0x7d, 0x6f, 0x83, 0x39, 0x37, 0x81, 0xbf, 0x86, 0x94, 0xc0, 0xf4, - 0x7c, 0xcf, 0xa4, 0xd1, 0x08, 0xfe, 0x22, 0x42, 0x2d, 0x7d, 0x1c, 0x6a, 0x53, 0x40, 0x60, 0x51, - 0x7f, 0x92, 0xca, 0xfd, 0x1c, 0x83, 0x3b, 0x37, 0x7a, 0x12, 0x3d, 0x82, 0x14, 0xf5, 0x4c, 0x36, - 0x0e, 0x38, 0x0d, 0x1f, 0x78, 0x09, 0x4f, 0x05, 0x22, 0x1a, 0xf1, 0x6a, 0x61, 0x34, 0x8b, 0xb7, - 0x8e, 0xa6, 0x12, 0x04, 0x51, 0x34, 0x46, 0x44, 0xa1, 0x16, 0xc4, 0x3d, 0xca, 0x89, 0x13, 0xb5, - 0x4f, 0x75, 0x37, 0x82, 0x2d, 0x7c, 0xcc, 0x5e, 0xa0, 0x7c, 0xbf, 0x8e, 0x35, 0x8f, 0xf2, 0x7d, - 0xeb, 0x5a, 0xab, 0xa9, 0x7f, 0x5f, 0xab, 0x7d, 0x09, 0x69, 0xab, 0x4f, 0x06, 0x94, 0x73, 0xe1, - 0x15, 0x0d, 0xb9, 0x69, 0xa7, 0xd4, 0x0f, 0xda, 0x91, 0x6a, 0xa6, 0xe9, 0xc0, 0xea, 0x4f, 0xa4, - 0xd7, 0xf6, 0x4d, 0xfc, 0x4f, 0xf7, 0x4d, 0xe2, 0x83, 0xfb, 0x26, 0xf7, 0x15, 0xc0, 0xf4, 0xa0, - 0x9b, 0xdb, 0x4f, 0xf9, 0xd0, 0xf6, 0x5b, 0x9c, 0xd9, 0x7e, 0xb9, 0x47, 0x10, 0x0f, 0xa1, 0x11, - 0x02, 0xd5, 0x16, 0x8d, 0xaa, 0xac, 0xc7, 0xe4, 0x40, 0x60, 0xf4, 0xf5, 0xe6, 0x1a, 0xc0, 0xf4, - 0xc7, 0x13, 0x4a, 0x82, 0x7a, 0xd0, 0xc2, 0x15, 0x7d, 0x01, 0x25, 0x20, 0xb6, 0xd7, 0x7e, 0xa1, - 0x2b, 0x9b, 0xdf, 0x29, 0x10, 0x0f, 0x37, 0x1c, 0x5a, 0x06, 0x68, 0x74, 0xc8, 0xee, 0xd3, 0x12, - 0xd9, 0xdd, 0x29, 0xe8, 0x0b, 0x82, 0xef, 0xb4, 0x49, 0xb9, 0x50, 0x24, 0xe5, 0xe2, 0xae, 0xae, - 0x08, 0xbe, 0xd6, 0x24, 0x3b, 0x3b, 0x65, 0xb2, 0xb3, 0xbb, 0xa3, 0x2f, 0x22, 0x80, 0x78, 0xa3, - 0x43, 0x1e, 0x97, 0x4a, 0x7a, 0x4c, 0xe8, 0x2a, 0x1d, 0x52, 0xde, 0x7e, 0x22, 0x6d, 0xd5, 0xc8, - 0xf6, 0xf1, 0x4e, 0x81, 0x3c, 0xd9, 0x2e, 0xe8, 0x9a, 0xb0, 0xad, 0xb4, 0x49, 0xb9, 0x58, 0xd2, - 0xe3, 0x42, 0xf7, 0x02, 0x93, 0x72, 0xb1, 0x20, 0xf9, 0xc4, 0xe6, 0xbf, 0x40, 0x93, 0xe3, 0x5d, - 0x28, 0x44, 0x78, 0xaf, 0x2a, 0x4d, 0x82, 0xb7, 0xf5, 0x85, 0xcd, 0x6f, 0x41, 0x93, 0xdb, 0x01, - 0xe9, 0xb0, 0xf4, 0xbc, 0xb5, 0xdf, 0x24, 0xb8, 0xf1, 0xb2, 0xd3, 0x68, 0x1f, 0xeb, 0x0b, 0x68, - 0x05, 0xd2, 0x52, 0x52, 0xa9, 0xd5, 0x1a, 0x47, 0xc7, 0xba, 0x82, 0x10, 0x2c, 0x77, 0x9a, 0xb5, - 0x56, 0x73, 0x6f, 0x1f, 0x1f, 0x36, 0xea, 0xa4, 0x73, 0xa4, 0x2f, 0xa2, 0x7b, 0xa0, 0xcf, 0xca, - 0xea, 0xad, 0x57, 0x4d, 0x3d, 0x26, 0xc0, 0xae, 0xd9, 0xa9, 0xc2, 0x77, 0xce, 0x4a, 0xab, 0x56, - 0xdf, 0xbc, 0x5f, 0x55, 0xde, 0xbe, 0x5f, 0x55, 0x7e, 0x7b, 0xbf, 0xaa, 0x7c, 0xf3, 0xf8, 0x36, - 0xff, 0x03, 0x4e, 0xe2, 0x52, 0x52, 0xfa, 0x3d, 0x00, 0x00, 0xff, 0xff, 0x15, 0x1b, 0xb4, 0x6a, - 0x46, 0x0c, 0x00, 0x00, + // 1311 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xcc, 0x57, 0x4b, 0x6f, 0xdb, 0xc6, + 0x16, 0x36, 0x2d, 0x52, 0x92, 0x8f, 0xfc, 0x60, 0x26, 0x09, 0xae, 0x6e, 0x12, 0xd8, 0x86, 0x70, + 0x2f, 0xae, 0x61, 0xdc, 0x5a, 0xb2, 0x14, 0xc7, 0x56, 0x8b, 0x06, 0xd0, 0xcb, 0x8d, 0x13, 0x5b, + 0x72, 0x46, 0x16, 0x52, 0x74, 0x33, 0xa0, 0xc9, 0xa1, 0x4c, 0x4b, 0x7c, 0x64, 0x34, 0xb2, 0xa5, + 0xae, 0xfb, 0x1b, 0xba, 0xee, 0xbe, 0xdb, 0x2e, 0xfa, 0x13, 0xb2, 0xcc, 0xa6, 0x5d, 0x64, 0x61, + 0x14, 0xf9, 0x25, 0xc5, 0x0c, 0xa9, 0x87, 0xe5, 0x36, 0x45, 0x9c, 0x2e, 0xba, 0xe2, 0x79, 0x7e, + 0x73, 0x66, 0xce, 0x4b, 0x82, 0x72, 0xdb, 0xe1, 0x67, 0xfd, 0xd3, 0x2d, 0xd3, 0x77, 0xb3, 0x27, + 0x67, 0xf4, 0xe4, 0xcc, 0xf1, 0xda, 0xbd, 0x3a, 0xe5, 0x97, 0x3e, 0xeb, 0x64, 0x39, 0xf7, 0xb2, + 0x46, 0xe0, 0x64, 0x03, 0xe6, 0x73, 0xdf, 0xf4, 0xbb, 0xd9, 0xae, 0xcf, 0x8c, 0x4b, 0xc3, 0x1b, + 0x7d, 0xb7, 0xa4, 0x02, 0x25, 0x22, 0xf6, 0xc1, 0x67, 0x53, 0x60, 0x6d, 0xbf, 0xed, 0x87, 0x8e, + 0xa7, 0x7d, 0x5b, 0x72, 0x92, 0x91, 0x54, 0xe8, 0x97, 0xf9, 0x55, 0x81, 0xe4, 0x11, 0xe5, 0x86, + 0x65, 0x70, 0x03, 0x15, 0x00, 0x5c, 0xdf, 0xea, 0x77, 0x0d, 0xee, 0xf8, 0x5e, 0x3a, 0xb5, 0xae, + 0x6c, 0x2c, 0xe7, 0xef, 0x6e, 0x8d, 0x0e, 0x3a, 0x1a, 0xab, 0xf0, 0x94, 0x19, 0x7a, 0x08, 0x0b, + 0xc2, 0x99, 0x30, 0x83, 0xd3, 0xf4, 0xe2, 0xba, 0xb2, 0xb1, 0x80, 0x93, 0x42, 0x80, 0x0d, 0x4e, + 0xd1, 0xbf, 0x21, 0x79, 0xea, 0xf0, 0x50, 0xb7, 0xb4, 0xae, 0x6c, 0x2c, 0xe1, 0xc4, 0xa9, 0xc3, + 0xa5, 0x6a, 0x0d, 0x52, 0xa6, 0x6f, 0x39, 0x5e, 0x3b, 0xd4, 0x2e, 0x4b, 0x4f, 0x08, 0x45, 0xd2, + 0xe0, 0x2e, 0x68, 0x36, 0x31, 0x3d, 0x9e, 0x5e, 0x91, 0x8e, 0xaa, 0x5d, 0xf1, 0x38, 0xfa, 0x1f, + 0xc4, 0x19, 0x6d, 0x8b, 0xf0, 0x74, 0x19, 0xde, 0xca, 0x38, 0x3c, 0x2c, 0xc5, 0x38, 0x52, 0x67, + 0x7e, 0x52, 0x60, 0xe5, 0x64, 0x50, 0xf1, 0x3d, 0xdb, 0x69, 0xf7, 0x59, 0x18, 0xea, 0x3f, 0xff, + 0x7e, 0x99, 0xef, 0x54, 0x40, 0x25, 0x93, 0x3b, 0x17, 0xf2, 0xf0, 0x71, 0x66, 0xea, 0x90, 0x30, + 0x82, 0x80, 0xd0, 0xbe, 0x93, 0x56, 0xd6, 0x95, 0x8d, 0xc5, 0xf2, 0xce, 0xbb, 0xab, 0xb5, 0xed, + 0xbf, 0xaa, 0x1b, 0xd3, 0x67, 0x34, 0xcb, 0x87, 0x01, 0xed, 0x6d, 0x95, 0x82, 0xa0, 0xd6, 0x3a, + 0xc0, 0x71, 0x23, 0x08, 0x6a, 0x7d, 0x47, 0xe0, 0x59, 0xf4, 0x42, 0xe2, 0xcd, 0xdf, 0x0a, 0xaf, + 0x4a, 0x2f, 0x24, 0x9e, 0x45, 0x2f, 0x04, 0xde, 0x4b, 0x48, 0x0a, 0x3c, 0xc3, 0xb2, 0x58, 0x3a, + 0x26, 0x01, 0x9f, 0xbc, 0xbb, 0x5a, 0xcb, 0x7f, 0x1c, 0x60, 0xc9, 0xb2, 0x18, 0x16, 0x71, 0x09, + 0x02, 0x61, 0x58, 0xf0, 0x2e, 0x3b, 0xa4, 0x47, 0x3a, 0x74, 0x98, 0x56, 0x6f, 0x85, 0x59, 0xbf, + 0xec, 0x34, 0x5f, 0xd0, 0x21, 0x4e, 0x78, 0x21, 0x81, 0x32, 0xb0, 0xc4, 0x06, 0xdb, 0xc4, 0x62, + 0xc4, 0xb7, 0xed, 0x1e, 0xe5, 0xb2, 0x06, 0x96, 0x70, 0x8a, 0x0d, 0xb6, 0xab, 0xac, 0x21, 0x45, + 0xe8, 0x3e, 0xc4, 0xd9, 0x20, 0x4f, 0x2c, 0x26, 0x93, 0xbd, 0x84, 0x35, 0x36, 0xc8, 0x57, 0x99, + 0xc8, 0x34, 0x1b, 0x10, 0x8b, 0x76, 0x8d, 0xe1, 0x28, 0xd3, 0x6c, 0x50, 0x15, 0x2c, 0xda, 0x80, + 0x84, 0x69, 0x93, 0xae, 0xd3, 0xe3, 0x32, 0xcb, 0xa9, 0xa9, 0xa2, 0xac, 0xec, 0x1f, 0x3a, 0x3d, + 0x8e, 0xe3, 0xa6, 0x2d, 0xbe, 0x53, 0xd5, 0xbb, 0xf2, 0xe1, 0xea, 0xfd, 0x71, 0x1e, 0x12, 0x47, + 0xb4, 0xd7, 0x33, 0xda, 0x14, 0xfd, 0x1f, 0x34, 0x97, 0x9c, 0x59, 0x4c, 0x66, 0x3e, 0x95, 0x5f, + 0x9a, 0x14, 0xec, 0xb3, 0x2a, 0x2e, 0x27, 0xdf, 0x5c, 0xad, 0xcd, 0xbd, 0xbd, 0x5a, 0x53, 0xb0, + 0xea, 0x3e, 0xb3, 0x18, 0xd2, 0x21, 0xe6, 0x3a, 0x66, 0x98, 0x55, 0x2c, 0x48, 0xf4, 0x04, 0x52, + 0xae, 0x61, 0x92, 0xc0, 0x18, 0x76, 0x7d, 0xc3, 0x92, 0xe9, 0x49, 0x4d, 0x97, 0x7d, 0xa9, 0x72, + 0x1c, 0xaa, 0x9e, 0xcd, 0x61, 0x70, 0x0d, 0x33, 0xe2, 0x50, 0x03, 0xee, 0x9d, 0xfb, 0x8e, 0x47, + 0x18, 0x7d, 0xdd, 0xa7, 0x3d, 0x3e, 0x06, 0x50, 0x25, 0xc0, 0xc3, 0x31, 0xc0, 0x73, 0xdf, 0xf1, + 0x70, 0x68, 0x33, 0x01, 0x42, 0xe7, 0x37, 0xa4, 0xe8, 0x10, 0xee, 0x4a, 0x40, 0xc3, 0x34, 0x69, + 0x30, 0xc1, 0xd3, 0x24, 0xde, 0x83, 0x6b, 0x78, 0x25, 0x69, 0x32, 0x81, 0xbb, 0x73, 0x3e, 0x2b, + 0x2c, 0x2f, 0x40, 0x22, 0x22, 0x33, 0x4d, 0x50, 0xc5, 0x5b, 0xa0, 0xff, 0x42, 0xdc, 0x25, 0x22, + 0xf5, 0xf2, 0xa9, 0x96, 0xf3, 0xcb, 0x93, 0x4b, 0x9e, 0x0c, 0x03, 0x8a, 0x35, 0x57, 0x7c, 0xd0, + 0x7f, 0x40, 0x73, 0x8d, 0x73, 0x9f, 0xc9, 0x47, 0xba, 0x66, 0x25, 0xa4, 0x38, 0x54, 0x66, 0x18, + 0xc0, 0xe4, 0x69, 0x44, 0x12, 0xec, 0x3f, 0x4c, 0xc2, 0xfe, 0x4c, 0x12, 0x6c, 0x91, 0x84, 0xfb, + 0x10, 0xb7, 0x49, 0xe0, 0x33, 0x2e, 0x8f, 0xd0, 0xb0, 0x66, 0x1f, 0xfb, 0x8c, 0x8b, 0x91, 0x60, + 0x33, 0xf7, 0x5a, 0x26, 0x16, 0x31, 0xd8, 0xcc, 0x1d, 0x5d, 0xe4, 0x17, 0x05, 0x54, 0x01, 0x88, + 0x5a, 0x53, 0xfd, 0x14, 0x36, 0xfc, 0xe7, 0xe2, 0x88, 0x4f, 0xed, 0xa9, 0xac, 0x88, 0xcb, 0xe4, + 0xac, 0x2b, 0xe3, 0x4a, 0x4d, 0x5d, 0x7d, 0xbf, 0xc2, 0x59, 0x77, 0xea, 0x1e, 0x9a, 0x2d, 0x04, + 0x93, 0x19, 0x15, 0x9b, 0x9a, 0xc1, 0x39, 0x81, 0xe2, 0x07, 0xbc, 0x97, 0x56, 0xd7, 0x63, 0xb3, + 0xb5, 0x54, 0xf1, 0x5d, 0xd7, 0xf0, 0xac, 0xb2, 0x2a, 0xa0, 0xb0, 0x66, 0x37, 0x02, 0xde, 0xcb, + 0x9c, 0x81, 0x26, 0x0f, 0x10, 0xd5, 0x69, 0x44, 0x57, 0x4a, 0x62, 0x41, 0xa2, 0x55, 0x48, 0x19, + 0x16, 0x23, 0x86, 0xd9, 0x11, 0x85, 0x26, 0xe3, 0x4a, 0xe2, 0x05, 0xc3, 0x62, 0x25, 0xb3, 0x83, + 0xe9, 0x6b, 0xe9, 0x61, 0x76, 0xe4, 0xf9, 0xc2, 0xc3, 0xec, 0x88, 0x81, 0x6c, 0x93, 0x80, 0x7a, + 0x62, 0x90, 0xca, 0x62, 0x4c, 0xe2, 0xa4, 0x7d, 0x1c, 0xf2, 0x99, 0x3d, 0x99, 0xb5, 0x28, 0x08, + 0xe1, 0x6c, 0x3a, 0x96, 0x3c, 0x6e, 0x09, 0x0b, 0x12, 0xa5, 0x21, 0x31, 0x7a, 0xfe, 0xb0, 0x45, + 0x46, 0x6c, 0xe6, 0xfb, 0x79, 0x40, 0x37, 0x4b, 0x19, 0xe1, 0xd9, 0xc9, 0x5b, 0x8c, 0x12, 0xf1, + 0x09, 0xd3, 0x17, 0xcf, 0x4e, 0xdf, 0xdb, 0x60, 0xce, 0x4c, 0xe0, 0xaf, 0x61, 0x41, 0x60, 0x7a, + 0xbe, 0x67, 0xd2, 0x68, 0x04, 0x7f, 0x11, 0xa1, 0x16, 0x3e, 0x0e, 0xb5, 0x2e, 0x20, 0xb0, 0xa8, + 0x3f, 0x49, 0x65, 0x7e, 0x8e, 0xc1, 0x9d, 0x1b, 0x3d, 0x89, 0x1e, 0xc1, 0x02, 0xf5, 0x4c, 0x36, + 0x0c, 0x38, 0x0d, 0x1f, 0x78, 0x11, 0x4f, 0x04, 0x22, 0x1a, 0xf1, 0x6a, 0x61, 0x34, 0xf3, 0xb7, + 0x8e, 0xa6, 0x14, 0x04, 0x51, 0x34, 0x46, 0x44, 0xa1, 0x06, 0xc4, 0x3d, 0xca, 0x89, 0x13, 0xb5, + 0x4f, 0x79, 0x2f, 0x82, 0xcd, 0x7d, 0xcc, 0x5e, 0xa0, 0xfc, 0xa0, 0x8a, 0x35, 0x8f, 0xf2, 0x03, + 0xeb, 0x5a, 0xab, 0xa9, 0x7f, 0x5f, 0xab, 0x3d, 0x85, 0x94, 0xd5, 0x25, 0x3d, 0xca, 0xb9, 0xf0, + 0x8a, 0x86, 0xdc, 0xa4, 0x53, 0xaa, 0x87, 0xcd, 0x48, 0x35, 0xd5, 0x74, 0x60, 0x75, 0x47, 0xd2, + 0x6b, 0xfb, 0x26, 0xfe, 0xa7, 0xfb, 0x26, 0xf1, 0xc1, 0x7d, 0x93, 0xf9, 0x0a, 0x60, 0x72, 0xd0, + 0xcd, 0xed, 0xa7, 0x7c, 0x68, 0xfb, 0xcd, 0x4f, 0x6d, 0xbf, 0xcc, 0x23, 0x88, 0x87, 0xd0, 0x08, + 0x81, 0x6a, 0x8b, 0x46, 0x55, 0xd6, 0x63, 0x72, 0x20, 0x30, 0xfa, 0x7a, 0x73, 0x0d, 0x60, 0xf2, + 0xe3, 0x09, 0x25, 0x41, 0x3d, 0x6c, 0xe0, 0x92, 0x3e, 0x87, 0x12, 0x10, 0xdb, 0x6f, 0xbe, 0xd0, + 0x95, 0xcd, 0x1f, 0x14, 0x88, 0x87, 0x1b, 0x0e, 0x2d, 0x03, 0xd4, 0x5a, 0x64, 0xef, 0x49, 0x81, + 0xec, 0xed, 0xe6, 0xf4, 0x39, 0xc1, 0xb7, 0x9a, 0xa4, 0x98, 0xcb, 0x93, 0x62, 0x7e, 0x4f, 0x57, + 0x04, 0x5f, 0xa9, 0x93, 0xdd, 0xdd, 0x22, 0xd9, 0xdd, 0xdb, 0xd5, 0xe7, 0x11, 0x40, 0xbc, 0xd6, + 0x22, 0x8f, 0x0b, 0x05, 0x3d, 0x26, 0x74, 0xa5, 0x16, 0x29, 0x6e, 0xef, 0x48, 0x5b, 0x35, 0xb2, + 0x7d, 0xbc, 0x9b, 0x23, 0x3b, 0xdb, 0x39, 0x5d, 0x13, 0xb6, 0xa5, 0x26, 0x29, 0xe6, 0x0b, 0x7a, + 0x5c, 0xda, 0x0a, 0x3a, 0x27, 0xf9, 0x2f, 0xc7, 0x7c, 0x81, 0x14, 0xf3, 0x3b, 0xfa, 0x53, 0xc1, + 0xbf, 0xc0, 0x63, 0x7d, 0x62, 0xf3, 0x5f, 0xa0, 0xc9, 0xf1, 0x2f, 0x14, 0x22, 0xfc, 0x57, 0xa5, + 0x3a, 0xc1, 0xdb, 0xfa, 0xdc, 0xe6, 0xb7, 0xa0, 0xc9, 0xed, 0x81, 0x74, 0x58, 0x7c, 0xde, 0x38, + 0xa8, 0x13, 0x5c, 0x7b, 0xd9, 0xaa, 0x35, 0x4f, 0xf4, 0x39, 0xb4, 0x02, 0x29, 0x29, 0x29, 0x55, + 0x2a, 0xb5, 0xe3, 0x13, 0x5d, 0x41, 0x08, 0x96, 0x5b, 0xf5, 0x4a, 0xa3, 0xbe, 0x7f, 0x80, 0x8f, + 0x6a, 0x55, 0xd2, 0x3a, 0xd6, 0xe7, 0xd1, 0x3d, 0xd0, 0xa7, 0x65, 0xd5, 0xc6, 0xab, 0xba, 0x1e, + 0x13, 0x60, 0xd7, 0xec, 0x54, 0xe1, 0x3b, 0x63, 0xa5, 0x95, 0xcb, 0x6f, 0xde, 0xaf, 0x2a, 0x6f, + 0xdf, 0xaf, 0x2a, 0xbf, 0xbd, 0x5f, 0x55, 0xbe, 0x79, 0x7c, 0x9b, 0xff, 0x09, 0xa7, 0x71, 0x29, + 0x29, 0xfc, 0x1e, 0x00, 0x00, 0xff, 0xff, 0xd4, 0xd0, 0x69, 0x31, 0x66, 0x0c, 0x00, 0x00, } diff --git a/api/protocol/lorawan/lorawan.proto b/api/protocol/lorawan/lorawan.proto index af866801a..b079ebcd4 100644 --- a/api/protocol/lorawan/lorawan.proto +++ b/api/protocol/lorawan/lorawan.proto @@ -57,12 +57,21 @@ message ActivationMetadata { enum Region { EU_863_870 = 0; + US_902_928 = 1; + CN_779_787 = 2; + EU_433 = 3; + AU_915_928 = 4; + CN_470_510 = 5; + AS_923 = 6; + AS_920_923 = 61; + AS_923_925 = 62; + KR_920_923 = 7; } diff --git a/core/band/band.go b/core/band/band.go index 80f8a5644..ba656acce 100644 --- a/core/band/band.go +++ b/core/band/band.go @@ -45,29 +45,29 @@ func (f *FrequencyPlan) GetTxPowerIndexFor(txPower int) (int, error) { // Guess the region based on frequency func Guess(frequency uint64) string { + // Join frequencies switch { - case frequency >= 863000000 && frequency <= 870000000: - return pb_lorawan.Region_EU_863_870.String() - case frequency >= 902300000 && frequency <= 914900000: - return pb_lorawan.Region_US_902_928.String() - case frequency >= 779500000 && frequency <= 786500000: - return pb_lorawan.Region_CN_779_787.String() - case frequency >= 433175000 && frequency <= 434665000: - return pb_lorawan.Region_EU_433.String() - case frequency == 923200000 || frequency == 923400000: + case frequency == 923200000 && frequency <= 923400000: + // not considering AS_920_923 and AS_923_925 because we're not sure return pb_lorawan.Region_AS_923.String() - case frequency >= 920900000 || frequency == 923300000: + case frequency == 922100000 || frequency == 922300000 || frequency == 922500000: return pb_lorawan.Region_KR_920_923.String() - case frequency >= 915200000 && frequency <= 927800000: - return pb_lorawan.Region_AU_915_928.String() - case frequency >= 470300000 && frequency <= 489300000: - return pb_lorawan.Region_CN_470_510.String() } + + // Existing Channels + if region, ok := channels[int(frequency)]; ok { + return region + } + + // Everything Else: not allowed return "" } // Get the frequency plan for the given region func Get(region string) (frequencyPlan FrequencyPlan, err error) { + if fp, ok := frequencyPlans[region]; ok { + return fp, nil + } switch region { case pb_lorawan.Region_EU_863_870.String(): frequencyPlan.Band, err = lora.GetConfig(lora.EU_863_870, false, lorawan.DwellTimeNoLimit) @@ -100,6 +100,38 @@ func Get(region string) (frequencyPlan FrequencyPlan, err error) { frequencyPlan.Band, err = lora.GetConfig(lora.CN_470_510, false, lorawan.DwellTimeNoLimit) case pb_lorawan.Region_AS_923.String(): frequencyPlan.Band, err = lora.GetConfig(lora.AS_923, false, lorawan.DwellTime400ms) + case pb_lorawan.Region_AS_920_923.String(): + frequencyPlan.Band, err = lora.GetConfig(lora.AS_923, false, lorawan.DwellTime400ms) + frequencyPlan.UplinkChannels = []lora.Channel{ + lora.Channel{Frequency: 923200000, DataRates: []int{0, 1, 2, 3, 4, 5}}, + lora.Channel{Frequency: 923400000, DataRates: []int{0, 1, 2, 3, 4, 5}}, + lora.Channel{Frequency: 922200000, DataRates: []int{0, 1, 2, 3, 4, 5}}, + lora.Channel{Frequency: 922400000, DataRates: []int{0, 1, 2, 3, 4, 5}}, + lora.Channel{Frequency: 922600000, DataRates: []int{0, 1, 2, 3, 4, 5}}, + lora.Channel{Frequency: 922800000, DataRates: []int{0, 1, 2, 3, 4, 5}}, + lora.Channel{Frequency: 923000000, DataRates: []int{0, 1, 2, 3, 4, 5}}, + lora.Channel{Frequency: 922000000, DataRates: []int{0, 1, 2, 3, 4, 5}}, + lora.Channel{Frequency: 922100000, DataRates: []int{6}}, + lora.Channel{Frequency: 921800000, DataRates: []int{7}}, + } + frequencyPlan.DownlinkChannels = frequencyPlan.UplinkChannels + frequencyPlan.CFList = &lorawan.CFList{922200000, 922400000, 922600000, 922800000, 923000000} + case pb_lorawan.Region_AS_923_925.String(): + frequencyPlan.Band, err = lora.GetConfig(lora.AS_923, false, lorawan.DwellTime400ms) + frequencyPlan.UplinkChannels = []lora.Channel{ + lora.Channel{Frequency: 923200000, DataRates: []int{0, 1, 2, 3, 4, 5}}, + lora.Channel{Frequency: 923400000, DataRates: []int{0, 1, 2, 3, 4, 5}}, + lora.Channel{Frequency: 923600000, DataRates: []int{0, 1, 2, 3, 4, 5}}, + lora.Channel{Frequency: 923800000, DataRates: []int{0, 1, 2, 3, 4, 5}}, + lora.Channel{Frequency: 924000000, DataRates: []int{0, 1, 2, 3, 4, 5}}, + lora.Channel{Frequency: 924200000, DataRates: []int{0, 1, 2, 3, 4, 5}}, + lora.Channel{Frequency: 924400000, DataRates: []int{0, 1, 2, 3, 4, 5}}, + lora.Channel{Frequency: 924600000, DataRates: []int{0, 1, 2, 3, 4, 5}}, + lora.Channel{Frequency: 924500000, DataRates: []int{6}}, + lora.Channel{Frequency: 924800000, DataRates: []int{7}}, + } + frequencyPlan.DownlinkChannels = frequencyPlan.UplinkChannels + frequencyPlan.CFList = &lorawan.CFList{923600000, 923800000, 924000000, 924200000, 924400000} case pb_lorawan.Region_KR_920_923.String(): frequencyPlan.Band, err = lora.GetConfig(lora.KR_920_923, false, lorawan.DwellTimeNoLimit) // TTN frequency plan includes extra channels next to the default channels: @@ -119,3 +151,33 @@ func Get(region string) (frequencyPlan FrequencyPlan, err error) { } return } + +var frequencyPlans map[string]FrequencyPlan +var channels map[int]string + +func init() { + frequencyPlans = make(map[string]FrequencyPlan) + channels = make(map[int]string) + for _, r := range []pb_lorawan.Region{ // ordering is important here + pb_lorawan.Region_EU_863_870, + pb_lorawan.Region_US_902_928, + pb_lorawan.Region_CN_779_787, + pb_lorawan.Region_EU_433, + pb_lorawan.Region_AS_923, + pb_lorawan.Region_AS_920_923, + pb_lorawan.Region_AS_923_925, + pb_lorawan.Region_KR_920_923, + pb_lorawan.Region_AU_915_928, + pb_lorawan.Region_CN_470_510, + } { + region := r.String() + frequencyPlans[region], _ = Get(region) + for _, ch := range frequencyPlans[region].UplinkChannels { + if len(ch.DataRates) > 1 { // ignore FSK channels + if _, ok := channels[ch.Frequency]; !ok { // ordering indicates priority + channels[ch.Frequency] = region + } + } + } + } +} diff --git a/core/band/band_test.go b/core/band/band_test.go index 6b0dbb419..13b76e126 100644 --- a/core/band/band_test.go +++ b/core/band/band_test.go @@ -19,7 +19,11 @@ func TestGuess(t *testing.T) { a.So(Guess(916800000), ShouldEqual, "AU_915_928") a.So(Guess(470300000), ShouldEqual, "CN_470_510") a.So(Guess(923200000), ShouldEqual, "AS_923") + a.So(Guess(922200000), ShouldEqual, "AS_920_923") + a.So(Guess(923600000), ShouldEqual, "AS_923_925") a.So(Guess(922100000), ShouldEqual, "KR_920_923") + + a.So(Guess(922100001), ShouldEqual, "") // Not allowed } func TestGet(t *testing.T) { @@ -74,6 +78,20 @@ func TestGet(t *testing.T) { a.So(fp.ADR, ShouldBeNil) } + { + fp, err := Get("AS_920_923") + a.So(err, ShouldBeNil) + a.So(fp.CFList, ShouldNotBeNil) + a.So(fp.ADR, ShouldBeNil) + } + + { + fp, err := Get("AS_923_925") + a.So(err, ShouldBeNil) + a.So(fp.CFList, ShouldNotBeNil) + a.So(fp.ADR, ShouldBeNil) + } + { fp, err := Get("KR_920_923") a.So(err, ShouldBeNil) From 8c8ba68602b0af580ebe8306de99e4bbb3b9511d Mon Sep 17 00:00:00 2001 From: Johan Stokking Date: Fri, 10 Mar 2017 14:22:41 +0100 Subject: [PATCH 02/13] Added device location to MQTT uplink documentation --- mqtt/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mqtt/README.md b/mqtt/README.md index 405422887..63eac7fb9 100644 --- a/mqtt/README.md +++ b/mqtt/README.md @@ -44,7 +44,10 @@ "altitude": 6 // Altitude of the gateway }, //...more if received by more gateways... - ] + ], + "latitude": 52.2345, // Latitude of the device + "longitude": 6.2345, // Longitude of the device + "altitude": 2 // Altitude of the device } } ``` From c360c23a8a8826635823049a263fe08503263d23 Mon Sep 17 00:00:00 2001 From: Hylke Visser Date: Thu, 9 Mar 2017 09:20:22 +0100 Subject: [PATCH 03/13] Make monitor streams for broker consistent with gateway --- api/monitor/monitor.go | 36 ++++++++++++++++++++++++++---------- api/monitor/monitor_test.go | 8 ++++++++ 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/api/monitor/monitor.go b/api/monitor/monitor.go index c6a7cd684..0beac0b7b 100644 --- a/api/monitor/monitor.go +++ b/api/monitor/monitor.go @@ -437,12 +437,28 @@ func (c *Client) NewBrokerStreams(id string, token string) GenericStream { } } + chUplink := make(chan *broker.DeduplicatedUplinkMessage, c.config.BufferSize) + chDownlink := make(chan *broker.DownlinkMessage, c.config.BufferSize) + + defer func() { + close(chUplink) + close(chDownlink) + }() + // Uplink stream uplink, err := cli.BrokerUplink(ctx) if err != nil { log.WithError(err).Warn("Could not set up BrokerUplink stream") } else { - go monitor("BrokerUplink", uplink) + s.mu.Lock() + s.uplink[server.name] = chUplink + s.mu.Unlock() + go func() { + monitor("BrokerUplink", uplink) + s.mu.Lock() + defer s.mu.Unlock() + delete(s.uplink, server.name) + }() } // Downlink stream @@ -450,17 +466,17 @@ func (c *Client) NewBrokerStreams(id string, token string) GenericStream { if err != nil { log.WithError(err).Warn("Could not set up BrokerDownlink stream") } else { - go monitor("BrokerDownlink", downlink) + s.mu.Lock() + s.downlink[server.name] = chDownlink + s.mu.Unlock() + go func() { + monitor("BrokerDownlink", downlink) + s.mu.Lock() + defer s.mu.Unlock() + delete(s.downlink, server.name) + }() } - chUplink := make(chan *broker.DeduplicatedUplinkMessage, c.config.BufferSize) - chDownlink := make(chan *broker.DownlinkMessage, c.config.BufferSize) - - s.mu.Lock() - s.uplink[server.name] = chUplink - s.downlink[server.name] = chDownlink - s.mu.Unlock() - log.Debug("Start handling Broker streams") defer log.Debug("Done handling Broker streams") for { diff --git a/api/monitor/monitor_test.go b/api/monitor/monitor_test.go index 7bf1746a4..1c5867a5b 100644 --- a/api/monitor/monitor_test.go +++ b/api/monitor/monitor_test.go @@ -40,6 +40,8 @@ func TestMonitor(t *testing.T) { cli.AddServer("tls-without-tls", lis.Addr().String()) + testLogger.Print(t) + cli.AddServer("test", lis.Addr().String()) time.Sleep(waitTime) defer func() { @@ -48,6 +50,8 @@ func TestMonitor(t *testing.T) { s.Stop() }() + testLogger.Print(t) + gtw := cli.NewGatewayStreams("test", "token") time.Sleep(waitTime) for i := 0; i < 20; i++ { @@ -64,6 +68,8 @@ func TestMonitor(t *testing.T) { a.So(server.metrics.downlinkMessages, ShouldEqual, 20) a.So(server.metrics.gatewayStatuses, ShouldEqual, 20) + testLogger.Print(t) + brk := cli.NewBrokerStreams("test", "token") time.Sleep(waitTime) brk.Send(&broker.DeduplicatedUplinkMessage{}) @@ -75,6 +81,8 @@ func TestMonitor(t *testing.T) { a.So(server.metrics.brokerUplinkMessages, ShouldEqual, 1) a.So(server.metrics.brokerDownlinkMessages, ShouldEqual, 1) + testLogger.Print(t) + cli.AddConn("test2", cli.serverConns[1].conn) brk = cli.NewBrokerStreams("test", "token") From 2ef5cb697b24c7cde64911eaa53f9b581516ecda Mon Sep 17 00:00:00 2001 From: Hylke Visser Date: Fri, 10 Mar 2017 15:20:32 +0100 Subject: [PATCH 04/13] Don't return on gRPC errors in send, disable streams before closing --- api/monitor/monitor.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/api/monitor/monitor.go b/api/monitor/monitor.go index 0beac0b7b..7a2f68bc2 100644 --- a/api/monitor/monitor.go +++ b/api/monitor/monitor.go @@ -441,6 +441,10 @@ func (c *Client) NewBrokerStreams(id string, token string) GenericStream { chDownlink := make(chan *broker.DownlinkMessage, c.config.BufferSize) defer func() { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.uplink, server.name) + delete(s.downlink, server.name) close(chUplink) close(chDownlink) }() @@ -486,12 +490,16 @@ func (c *Client) NewBrokerStreams(id string, token string) GenericStream { case msg := <-chUplink: if err := uplink.Send(msg); err != nil { log.WithError(err).Warn("Could not send UplinkMessage to monitor") - return + if err == restartstream.ErrStreamClosed { + return + } } case msg := <-chDownlink: if err := downlink.Send(msg); err != nil { log.WithError(err).Warn("Could not send DownlinkMessage to monitor") - return + if err == restartstream.ErrStreamClosed { + return + } } } } From 308469919cb04ae69e82a3653019d7018448903a Mon Sep 17 00:00:00 2001 From: Hylke Visser Date: Fri, 10 Mar 2017 15:30:21 +0100 Subject: [PATCH 05/13] Update go-utils --- vendor/vendor.json | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/vendor/vendor.json b/vendor/vendor.json index 3194e0d55..2b8b89621 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -77,68 +77,68 @@ { "checksumSHA1": "x795Q87cyvAAeuxXxft5iwd1Los=", "path": "github.com/TheThingsNetwork/go-utils/backoff", - "revision": "7571e4b271c4c2676e7a2d917c4b102c2bd1674b", - "revisionTime": "2017-03-03T09:55:16Z" + "revision": "4555a883db1fba5c15ee20e714a71372d653c886", + "revisionTime": "2017-03-10T14:29:42Z" }, { "checksumSHA1": "Dl7PzR7MCkAujv/tceQkrTzVq0U=", "path": "github.com/TheThingsNetwork/go-utils/encoding", - "revision": "7571e4b271c4c2676e7a2d917c4b102c2bd1674b", - "revisionTime": "2017-03-03T09:55:16Z" + "revision": "4555a883db1fba5c15ee20e714a71372d653c886", + "revisionTime": "2017-03-10T14:29:42Z" }, { "checksumSHA1": "Q+Nny6VBPlH007VXteMBuMPYikI=", "path": "github.com/TheThingsNetwork/go-utils/grpc/interceptor", - "revision": "7571e4b271c4c2676e7a2d917c4b102c2bd1674b", - "revisionTime": "2017-03-03T09:55:16Z" + "revision": "4555a883db1fba5c15ee20e714a71372d653c886", + "revisionTime": "2017-03-10T14:29:42Z" }, { - "checksumSHA1": "fCoHaJWT0nINfQgUlLM86ztWe4M=", + "checksumSHA1": "/0L+3YQZjPiYtK03Th/ThuUjNN4=", "path": "github.com/TheThingsNetwork/go-utils/grpc/restartstream", - "revision": "7571e4b271c4c2676e7a2d917c4b102c2bd1674b", - "revisionTime": "2017-03-03T09:55:16Z" + "revision": "4555a883db1fba5c15ee20e714a71372d653c886", + "revisionTime": "2017-03-10T14:29:42Z" }, { "checksumSHA1": "T7iFQUlCUAv4cJNDZC0//46Nbio=", "path": "github.com/TheThingsNetwork/go-utils/handlers/cli", - "revision": "7571e4b271c4c2676e7a2d917c4b102c2bd1674b", - "revisionTime": "2017-03-03T09:55:16Z" + "revision": "4555a883db1fba5c15ee20e714a71372d653c886", + "revisionTime": "2017-03-10T14:29:42Z" }, { "checksumSHA1": "aXt7ZSqIfsHWBbJPgHFjqtyxyQ0=", "path": "github.com/TheThingsNetwork/go-utils/log", - "revision": "7571e4b271c4c2676e7a2d917c4b102c2bd1674b", - "revisionTime": "2017-03-03T09:55:16Z" + "revision": "4555a883db1fba5c15ee20e714a71372d653c886", + "revisionTime": "2017-03-10T14:29:42Z" }, { "checksumSHA1": "RdI5upcV6MHSjr5Y9zogYvbeURw=", "path": "github.com/TheThingsNetwork/go-utils/log/apex", - "revision": "7571e4b271c4c2676e7a2d917c4b102c2bd1674b", - "revisionTime": "2017-03-03T09:55:16Z" + "revision": "4555a883db1fba5c15ee20e714a71372d653c886", + "revisionTime": "2017-03-10T14:29:42Z" }, { "checksumSHA1": "sQ0vy3MCGY1WgK9xldn1V6pMeZk=", "path": "github.com/TheThingsNetwork/go-utils/log/grpc", - "revision": "7571e4b271c4c2676e7a2d917c4b102c2bd1674b", - "revisionTime": "2017-03-03T09:55:16Z" + "revision": "4555a883db1fba5c15ee20e714a71372d653c886", + "revisionTime": "2017-03-10T14:29:42Z" }, { "checksumSHA1": "2/v0SMyHM5vgImOb1BEEDWeXZEY=", "path": "github.com/TheThingsNetwork/go-utils/pseudorandom", - "revision": "7571e4b271c4c2676e7a2d917c4b102c2bd1674b", - "revisionTime": "2017-03-03T09:55:16Z" + "revision": "4555a883db1fba5c15ee20e714a71372d653c886", + "revisionTime": "2017-03-10T14:29:42Z" }, { "checksumSHA1": "iYa+qSqzqZwpmoibM8/1X+aC3sI=", "path": "github.com/TheThingsNetwork/go-utils/random", - "revision": "7571e4b271c4c2676e7a2d917c4b102c2bd1674b", - "revisionTime": "2017-03-03T09:55:16Z" + "revision": "4555a883db1fba5c15ee20e714a71372d653c886", + "revisionTime": "2017-03-10T14:29:42Z" }, { "checksumSHA1": "kLFTtAVcjZbHXybodGAqJ8wxflY=", "path": "github.com/TheThingsNetwork/go-utils/roots", - "revision": "7571e4b271c4c2676e7a2d917c4b102c2bd1674b", - "revisionTime": "2017-03-03T09:55:16Z" + "revision": "4555a883db1fba5c15ee20e714a71372d653c886", + "revisionTime": "2017-03-10T14:29:42Z" }, { "checksumSHA1": "EZ0pNaUAiIbJuT5c0Sew85egLgw=", From ce70a8cd6045c3815e43a689aeb068fef75d8807 Mon Sep 17 00:00:00 2001 From: Hylke Visser Date: Fri, 10 Mar 2017 16:16:42 +0100 Subject: [PATCH 06/13] FATAL if database connection fails on start --- cmd/discovery.go | 4 +++- cmd/handler.go | 4 +++- cmd/networkserver.go | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cmd/discovery.go b/cmd/discovery.go index 4e0ab20c7..bd7a6faef 100644 --- a/cmd/discovery.go +++ b/cmd/discovery.go @@ -48,7 +48,9 @@ var discoveryCmd = &cobra.Command{ DB: viper.GetInt("discovery.redis-db"), }) - connectRedis(client) + if err := connectRedis(client); err != nil { + ctx.WithError(err).Fatal("Could not initialize database connection") + } // Component component, err := component.New(ttnlog.Get(), "discovery", fmt.Sprintf("%s:%d", "localhost", viper.GetInt("discovery.server-port"))) diff --git a/cmd/handler.go b/cmd/handler.go index 1871a4fc2..c82011dad 100644 --- a/cmd/handler.go +++ b/cmd/handler.go @@ -52,7 +52,9 @@ var handlerCmd = &cobra.Command{ DB: viper.GetInt("handler.redis-db"), }) - connectRedis(client) + if err := connectRedis(client); err != nil { + ctx.WithError(err).Fatal("Could not initialize database connection") + } // Component component, err := component.New(ttnlog.Get(), "handler", fmt.Sprintf("%s:%d", viper.GetString("handler.server-address-announce"), viper.GetInt("handler.server-port"))) diff --git a/cmd/networkserver.go b/cmd/networkserver.go index 48217e6c5..2e1ea5c43 100644 --- a/cmd/networkserver.go +++ b/cmd/networkserver.go @@ -43,7 +43,9 @@ var networkserverCmd = &cobra.Command{ DB: viper.GetInt("networkserver.redis-db"), }) - connectRedis(client) + if err := connectRedis(client); err != nil { + ctx.WithError(err).Fatal("Could not initialize database connection") + } // Component component, err := component.New(ttnlog.Get(), "networkserver", fmt.Sprintf("%s:%d", viper.GetString("networkserver.server-address-announce"), viper.GetInt("networkserver.server-port"))) From 9034daf7aff30c730eb9f9aaedbfffe4848a7b6e Mon Sep 17 00:00:00 2001 From: Hylke Visser Date: Thu, 9 Mar 2017 17:05:52 +0100 Subject: [PATCH 07/13] Add Token RPC Credentials --- api/auth/auth.go | 49 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 api/auth/auth.go diff --git a/api/auth/auth.go b/api/auth/auth.go new file mode 100644 index 000000000..8b45a56b3 --- /dev/null +++ b/api/auth/auth.go @@ -0,0 +1,49 @@ +package auth + +import ( + "github.com/TheThingsNetwork/ttn/api" + "golang.org/x/net/context" + "google.golang.org/grpc" +) + +const tokenKey = "token" + +// TokenCredentials RPC Credentials +type TokenCredentials struct { + token string + tokenFunc func(id string) string +} + +// WithStaticToken injects a static token on each request +func WithStaticToken(token string) *TokenCredentials { + return &TokenCredentials{token: token} +} + +// WithTokenFunc returns TokenCredentials that execute the tokenFunc on each request +func WithTokenFunc(tokenFunc func(id string) string) *TokenCredentials { + return &TokenCredentials{tokenFunc: tokenFunc} +} + +// RequireTransportSecurity implements credentials.PerRPCCredentials +func (c *TokenCredentials) RequireTransportSecurity() bool { return true } + +// GetRequestMetadata implements credentials.PerRPCCredentials +func (c *TokenCredentials) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { + token, _ := api.TokenFromContext(ctx) + if token != "" { + return map[string]string{tokenKey: token}, nil + } + if c.tokenFunc != nil { + id, _ := api.IDFromContext(ctx) + return map[string]string{tokenKey: c.tokenFunc(id)}, nil + } + if c.token != "" { + return map[string]string{tokenKey: c.token}, nil + } + return map[string]string{tokenKey: ""}, nil +} + +// DialOption returns a DialOption for the TokenCredentials +func (c *TokenCredentials) DialOption() grpc.DialOption { + return grpc.WithPerRPCCredentials(c) +} From 84ef1aafd5f1718b14836a3f6927b9e928f6b061 Mon Sep 17 00:00:00 2001 From: Hylke Visser Date: Fri, 10 Mar 2017 18:18:15 +0100 Subject: [PATCH 08/13] Don't HMSET if the only field is updated_at --- core/discovery/announcement/announcement.go | 5 +++++ core/handler/application/application.go | 5 +++++ core/handler/device/device.go | 5 +++++ core/networkserver/device/device.go | 5 +++++ core/storage/redis_map_store.go | 3 +++ 5 files changed, 23 insertions(+) diff --git a/core/discovery/announcement/announcement.go b/core/discovery/announcement/announcement.go index 446a598ad..3d6caeaa7 100644 --- a/core/discovery/announcement/announcement.go +++ b/core/discovery/announcement/announcement.go @@ -174,6 +174,11 @@ func (a Announcement) ChangedFields() (changed []string) { changed = append(changed, field.Name()) } } + + if len(changed) == 1 && changed[0] == "UpdatedAt" { + return []string{} + } + return } diff --git a/core/handler/application/application.go b/core/handler/application/application.go index dcd14e8d4..e82cb46d7 100644 --- a/core/handler/application/application.go +++ b/core/handler/application/application.go @@ -62,5 +62,10 @@ func (a Application) ChangedFields() (changed []string) { changed = append(changed, field.Name()) } } + + if len(changed) == 1 && changed[0] == "UpdatedAt" { + return []string{} + } + return } diff --git a/core/handler/device/device.go b/core/handler/device/device.go index 70e606383..78b80300a 100644 --- a/core/handler/device/device.go +++ b/core/handler/device/device.go @@ -84,6 +84,11 @@ func (d Device) ChangedFields() (changed []string) { changed = append(changed, field.Name()) } } + + if len(changed) == 1 && changed[0] == "UpdatedAt" { + return []string{} + } + return } diff --git a/core/networkserver/device/device.go b/core/networkserver/device/device.go index 1ded37a92..227921993 100644 --- a/core/networkserver/device/device.go +++ b/core/networkserver/device/device.go @@ -83,5 +83,10 @@ func (d Device) ChangedFields() (changed []string) { changed = append(changed, field.Name()) } } + + if len(changed) == 1 && changed[0] == "UpdatedAt" { + return []string{} + } + return } diff --git a/core/storage/redis_map_store.go b/core/storage/redis_map_store.go index c34e8e881..d1a76bbca 100644 --- a/core/storage/redis_map_store.go +++ b/core/storage/redis_map_store.go @@ -238,6 +238,9 @@ func (s *RedisMapStore) Update(key string, value interface{}, properties ...stri if len(properties) == 0 { if i, ok := value.(ChangedFielder); ok { properties = i.ChangedFields() + if len(properties) == 0 { + return nil + } } } From 0b274066d54852fde7de34999b8cebbae1b47f50 Mon Sep 17 00:00:00 2001 From: Hylke Visser Date: Mon, 13 Mar 2017 08:31:38 +0100 Subject: [PATCH 09/13] Add go_import_path to travis.yml to make build work in forks --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index d0d60462f..3127efc08 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,7 @@ language: go +go_import_path: github.com/TheThingsNetwork/ttn + sudo: required services: From 3452fabd084c0f5d735abe1787ef33a0deedb09c Mon Sep 17 00:00:00 2001 From: Hylke Visser Date: Mon, 13 Mar 2017 12:09:55 +0100 Subject: [PATCH 10/13] Update Travis config - Use go 1.8 - Cache vendors --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3127efc08..f1f95a3f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,11 @@ services: - docker go: - - 1.7 + - 1.8 + +cache: + directories: + - vendor install: - make deps From 970477bf390b79e51c0299d09f27bfbc19fa176b Mon Sep 17 00:00:00 2001 From: Hylke Visser Date: Mon, 13 Mar 2017 10:31:49 +0100 Subject: [PATCH 11/13] Move common Redis functionality to base struct --- core/storage/redis_kv_store.go | 72 ++++------------ core/storage/redis_kv_store_test.go | 18 ++-- core/storage/redis_map_store.go | 119 +++++++++------------------ core/storage/redis_map_store_test.go | 20 +++-- core/storage/redis_queue_store.go | 35 ++------ core/storage/redis_set_store.go | 35 ++------ core/storage/redis_store.go | 68 +++++++++++++++ 7 files changed, 155 insertions(+), 212 deletions(-) create mode 100644 core/storage/redis_store.go diff --git a/core/storage/redis_kv_store.go b/core/storage/redis_kv_store.go index b2bb251be..3e41e8e35 100644 --- a/core/storage/redis_kv_store.go +++ b/core/storage/redis_kv_store.go @@ -8,14 +8,12 @@ import ( "strings" "github.com/TheThingsNetwork/ttn/utils/errors" - "gopkg.in/redis.v5" ) // RedisKVStore stores arbitrary data in Redis type RedisKVStore struct { - prefix string - client *redis.Client + *RedisStore } // NewRedisKVStore creates a new RedisKVStore @@ -24,8 +22,7 @@ func NewRedisKVStore(client *redis.Client, prefix string) *RedisKVStore { prefix += ":" } return &RedisKVStore{ - client: client, - prefix: prefix, + RedisStore: NewRedisStore(client, prefix), } } @@ -77,24 +74,9 @@ func (s *RedisKVStore) GetAll(keys []string, options *ListOptions) (map[string]s // List all results matching the selector, prepending the prefix to the selector if necessary func (s *RedisKVStore) List(selector string, options *ListOptions) (map[string]string, error) { - if selector == "" { - selector = "*" - } - if !strings.HasPrefix(selector, s.prefix) { - selector = s.prefix + selector - } - var allKeys []string - var cursor uint64 - for { - keys, next, err := s.client.Scan(cursor, selector, 0).Result() - if err != nil { - return nil, err - } - allKeys = append(allKeys, keys...) - cursor = next - if cursor == 0 { - break - } + allKeys, err := s.Keys(selector) + if err != nil { + return nil, err } return s.GetAll(allKeys, options) } @@ -114,12 +96,20 @@ func (s *RedisKVStore) Get(key string) (string, error) { return result, nil } +// Set a record, prepending the prefix to the key if necessary +func (s *RedisKVStore) Set(key string, value string) error { + if !strings.HasPrefix(key, s.prefix) { + key = s.prefix + key + } + return s.client.Set(key, value, 0).Err() +} + // Create a new record, prepending the prefix to the key if necessary +// This function returns an error if the record already exists func (s *RedisKVStore) Create(key string, value string) error { if !strings.HasPrefix(key, s.prefix) { key = s.prefix + key } - err := s.client.Watch(func(tx *redis.Tx) error { exists, err := tx.Exists(key).Result() if err != nil { @@ -140,16 +130,15 @@ func (s *RedisKVStore) Create(key string, value string) error { if err != nil { return err } - return nil } // Update an existing record, prepending the prefix to the key if necessary +// This function returns an error if the record does not exist func (s *RedisKVStore) Update(key string, value string) error { if !strings.HasPrefix(key, s.prefix) { key = s.prefix + key } - err := s.client.Watch(func(tx *redis.Tx) error { exists, err := tx.Exists(key).Result() if err != nil { @@ -170,36 +159,5 @@ func (s *RedisKVStore) Update(key string, value string) error { if err != nil { return err } - - return nil -} - -// Delete an existing record, prepending the prefix to the key if necessary -func (s *RedisKVStore) Delete(key string) error { - if !strings.HasPrefix(key, s.prefix) { - key = s.prefix + key - } - - err := s.client.Watch(func(tx *redis.Tx) error { - exists, err := tx.Exists(key).Result() - if err != nil { - return err - } - if !exists { - return errors.NewErrNotFound(key) - } - _, err = tx.Pipelined(func(pipe *redis.Pipeline) error { - pipe.Del(key) - return nil - }) - if err != nil { - return err - } - return nil - }, key) - if err != nil { - return err - } - return nil } diff --git a/core/storage/redis_kv_store_test.go b/core/storage/redis_kv_store_test.go index 721934163..b4adae1fc 100644 --- a/core/storage/redis_kv_store_test.go +++ b/core/storage/redis_kv_store_test.go @@ -105,23 +105,27 @@ func TestRedisKVStore(t *testing.T) { a.So(err, ShouldNotBeNil) } - // Update Existing + // Set { - err := s.Update("test", "updated") + err := s.Set("test", "other") a.So(err, ShouldBeNil) name, err := c.Get("test-redis-kv-store:test").Result() a.So(err, ShouldBeNil) - a.So(name, ShouldEqual, "updated") + a.So(name, ShouldEqual, "other") } - // Delete Non-Existing + // Update Existing { - err := s.Delete("not-there") - a.So(err, ShouldNotBeNil) + err := s.Update("test", "updated") + a.So(err, ShouldBeNil) + + name, err := c.Get("test-redis-kv-store:test").Result() + a.So(err, ShouldBeNil) + a.So(name, ShouldEqual, "updated") } - // Delete Existing + // Delete { err := s.Delete("test") a.So(err, ShouldBeNil) diff --git a/core/storage/redis_map_store.go b/core/storage/redis_map_store.go index d1a76bbca..554337c62 100644 --- a/core/storage/redis_map_store.go +++ b/core/storage/redis_map_store.go @@ -14,8 +14,7 @@ import ( // RedisMapStore stores structs as HMaps in Redis type RedisMapStore struct { - prefix string - client *redis.Client + *RedisStore encoder func(input interface{}, properties ...string) (map[string]string, error) decoder func(input map[string]string) (output interface{}, err error) migrations map[string]MigrateFunction @@ -27,8 +26,7 @@ func NewRedisMapStore(client *redis.Client, prefix string) *RedisMapStore { prefix += ":" } return &RedisMapStore{ - client: client, - prefix: prefix, + RedisStore: NewRedisStore(client, prefix), migrations: make(map[string]MigrateFunction), } } @@ -107,24 +105,9 @@ func (s *RedisMapStore) GetAll(keys []string, options *ListOptions) ([]interface // List all results matching the selector, prepending the prefix to the selector if necessary func (s *RedisMapStore) List(selector string, options *ListOptions) ([]interface{}, error) { - if selector == "" { - selector = "*" - } - if !strings.HasPrefix(selector, s.prefix) { - selector = s.prefix + selector - } - var allKeys []string - var cursor uint64 - for { - keys, next, err := s.client.Scan(cursor, selector, 0).Result() - if err != nil { - return nil, err - } - allKeys = append(allKeys, keys...) - cursor = next - if cursor == 0 { - break - } + allKeys, err := s.Keys(selector) + if err != nil { + return nil, err } return s.GetAll(allKeys, options) } @@ -181,30 +164,53 @@ type ChangedFielder interface { ChangedFields() []string } -// Create a new record, prepending the prefix to the key if necessary, optionally setting only the given properties -func (s *RedisMapStore) Create(key string, value interface{}, properties ...string) error { +func (s *RedisMapStore) prepare(key string, value interface{}, properties ...string) (fullKey string, vmap map[string]string, err error) { if !strings.HasPrefix(key, s.prefix) { - key = s.prefix + key + fullKey = s.prefix + key } - if len(properties) == 0 { if i, ok := value.(ChangedFielder); ok { properties = i.ChangedFields() + if len(properties) == 0 { + return + } } } + vmap, err = s.encoder(value, properties...) + if err != nil { + return + } + if len(vmap) == 0 { + return + } + if v, ok := value.(hasDBVersion); ok { + vmap[VersionKey] = v.DBVersion() + } + return +} - vmap, err := s.encoder(value, properties...) +// Set a record, prepending the prefix to the key if necessary, optionally setting only the given properties +func (s *RedisMapStore) Set(key string, value interface{}, properties ...string) error { + key, vmap, err := s.prepare(key, value, properties...) if err != nil { return err } if len(vmap) == 0 { return nil } + return s.client.HMSet(key, vmap).Err() +} - if v, ok := value.(hasDBVersion); ok { - vmap[VersionKey] = v.DBVersion() +// Create a new record, prepending the prefix to the key if necessary, optionally setting only the given properties +// This function returns an error if the record already exists +func (s *RedisMapStore) Create(key string, value interface{}, properties ...string) error { + key, vmap, err := s.prepare(key, value, properties...) + if err != nil { + return err + } + if len(vmap) == 0 { + return nil } - err = s.client.Watch(func(tx *redis.Tx) error { exists, err := tx.Exists(key).Result() if err != nil { @@ -225,37 +231,19 @@ func (s *RedisMapStore) Create(key string, value interface{}, properties ...stri if err != nil { return err } - return nil } // Update an existing record, prepending the prefix to the key if necessary, optionally setting only the given properties +// This function returns an error if the record does not exist func (s *RedisMapStore) Update(key string, value interface{}, properties ...string) error { - if !strings.HasPrefix(key, s.prefix) { - key = s.prefix + key - } - - if len(properties) == 0 { - if i, ok := value.(ChangedFielder); ok { - properties = i.ChangedFields() - if len(properties) == 0 { - return nil - } - } - } - - vmap, err := s.encoder(value, properties...) + key, vmap, err := s.prepare(key, value, properties...) if err != nil { return err } if len(vmap) == 0 { return nil } - - if v, ok := value.(hasDBVersion); ok { - vmap[VersionKey] = v.DBVersion() - } - err = s.client.Watch(func(tx *redis.Tx) error { exists, err := tx.Exists(key).Result() if err != nil { @@ -276,36 +264,5 @@ func (s *RedisMapStore) Update(key string, value interface{}, properties ...stri if err != nil { return err } - - return nil -} - -// Delete an existing record, prepending the prefix to the key if necessary -func (s *RedisMapStore) Delete(key string) error { - if !strings.HasPrefix(key, s.prefix) { - key = s.prefix + key - } - - err := s.client.Watch(func(tx *redis.Tx) error { - exists, err := tx.Exists(key).Result() - if err != nil { - return err - } - if !exists { - return errors.NewErrNotFound(key) - } - _, err = tx.Pipelined(func(pipe *redis.Pipeline) error { - pipe.Del(key) - return nil - }) - if err != nil { - return err - } - return nil - }, key) - if err != nil { - return err - } - return nil } diff --git a/core/storage/redis_map_store_test.go b/core/storage/redis_map_store_test.go index 1cd1ccbc1..ca5a10b4b 100644 --- a/core/storage/redis_map_store_test.go +++ b/core/storage/redis_map_store_test.go @@ -139,6 +139,18 @@ func TestRedisMapStore(t *testing.T) { a.So(err, ShouldNotBeNil) } + // Set + { + err := s.Set("test", &testRedisStruct{ + Name: "Other Name", + }, "Name") + a.So(err, ShouldBeNil) + + name, err := c.HGet("test-redis-map-store:test", "name").Result() + a.So(err, ShouldBeNil) + a.So(name, ShouldEqual, "Other Name") + } + // Update Existing { err := s.Update("test", &testRedisStruct{ @@ -151,13 +163,7 @@ func TestRedisMapStore(t *testing.T) { a.So(name, ShouldEqual, "New Name") } - // Delete Non-Existing - { - err := s.Delete("not-there") - a.So(err, ShouldNotBeNil) - } - - // Delete Existing + // Delete { err := s.Delete("test") a.So(err, ShouldBeNil) diff --git a/core/storage/redis_queue_store.go b/core/storage/redis_queue_store.go index 7a4eedfca..109357717 100644 --- a/core/storage/redis_queue_store.go +++ b/core/storage/redis_queue_store.go @@ -12,8 +12,7 @@ import ( // RedisQueueStore stores queues in Redis type RedisQueueStore struct { - prefix string - client *redis.Client + *RedisStore } // NewRedisQueueStore creates a new RedisQueueStore @@ -22,8 +21,7 @@ func NewRedisQueueStore(client *redis.Client, prefix string) *RedisQueueStore { prefix += ":" } return &RedisQueueStore{ - client: client, - prefix: prefix, + RedisStore: NewRedisStore(client, prefix), } } @@ -75,24 +73,9 @@ func (s *RedisQueueStore) GetAll(keys []string, options *ListOptions) (map[strin // List all results matching the selector, prepending the prefix to the selector if necessary func (s *RedisQueueStore) List(selector string, options *ListOptions) (map[string][]string, error) { - if selector == "" { - selector = "*" - } - if !strings.HasPrefix(selector, s.prefix) { - selector = s.prefix + selector - } - var allKeys []string - var cursor uint64 - for { - keys, next, err := s.client.Scan(cursor, selector, 0).Result() - if err != nil { - return nil, err - } - allKeys = append(allKeys, keys...) - cursor = next - if cursor == 0 { - break - } + allKeys, err := s.Keys(selector) + if err != nil { + return nil, err } return s.GetAll(allKeys, options) } @@ -197,11 +180,3 @@ func (s *RedisQueueStore) Trim(key string, length int) error { } return err } - -// Delete the entire queue -func (s *RedisQueueStore) Delete(key string) error { - if !strings.HasPrefix(key, s.prefix) { - key = s.prefix + key - } - return s.client.Del(key).Err() -} diff --git a/core/storage/redis_set_store.go b/core/storage/redis_set_store.go index cdd90322a..ad291e8e7 100644 --- a/core/storage/redis_set_store.go +++ b/core/storage/redis_set_store.go @@ -14,8 +14,7 @@ import ( // RedisSetStore stores sets in Redis type RedisSetStore struct { - prefix string - client *redis.Client + *RedisStore } // NewRedisSetStore creates a new RedisSetStore @@ -24,8 +23,7 @@ func NewRedisSetStore(client *redis.Client, prefix string) *RedisSetStore { prefix += ":" } return &RedisSetStore{ - client: client, - prefix: prefix, + RedisStore: NewRedisStore(client, prefix), } } @@ -78,24 +76,9 @@ func (s *RedisSetStore) GetAll(keys []string, options *ListOptions) (map[string] // List all results matching the selector, prepending the prefix to the selector if necessary func (s *RedisSetStore) List(selector string, options *ListOptions) (map[string][]string, error) { - if selector == "" { - selector = "*" - } - if !strings.HasPrefix(selector, s.prefix) { - selector = s.prefix + selector - } - var allKeys []string - var cursor uint64 - for { - keys, next, err := s.client.Scan(cursor, selector, 0).Result() - if err != nil { - return nil, err - } - allKeys = append(allKeys, keys...) - cursor = next - if cursor == 0 { - break - } + allKeys, err := s.Keys(selector) + if err != nil { + return nil, err } return s.GetAll(allKeys, options) } @@ -148,11 +131,3 @@ func (s *RedisSetStore) Remove(key string, values ...string) error { } return s.client.SRem(key, valuesI...).Err() } - -// Delete the entire set -func (s *RedisSetStore) Delete(key string) error { - if !strings.HasPrefix(key, s.prefix) { - key = s.prefix + key - } - return s.client.Del(key).Err() -} diff --git a/core/storage/redis_store.go b/core/storage/redis_store.go new file mode 100644 index 000000000..c50a8d8c4 --- /dev/null +++ b/core/storage/redis_store.go @@ -0,0 +1,68 @@ +// Copyright © 2017 The Things Network +// Use of this source code is governed by the MIT license that can be found in the LICENSE file. + +package storage + +import ( + "strings" + + "gopkg.in/redis.v5" +) + +// RedisStore is the base of more specialized stores +type RedisStore struct { + prefix string + client *redis.Client +} + +// NewRedisStore creates a new RedisStore +func NewRedisStore(client *redis.Client, prefix string) *RedisStore { + if !strings.HasSuffix(prefix, ":") { + prefix += ":" + } + return &RedisStore{ + client: client, + prefix: prefix, + } +} + +// Keys matching the selector, prepending the prefix to the selector if necessary +func (s *RedisStore) Keys(selector string) ([]string, error) { + if selector == "" { + selector = "*" + } + if !strings.HasPrefix(selector, s.prefix) { + selector = s.prefix + selector + } + var allKeys []string + var cursor uint64 + for { + keys, next, err := s.client.Scan(cursor, selector, 0).Result() + if err != nil { + return nil, err + } + allKeys = append(allKeys, keys...) + cursor = next + if cursor == 0 { + break + } + } + return allKeys, nil +} + +// Count the results matching the selector +func (s *RedisStore) Count(selector string) (int, error) { + allKeys, err := s.Keys(selector) + if err != nil { + return 0, err + } + return len(allKeys), nil +} + +// Delete an existing record, prepending the prefix to the key if necessary +func (s *RedisStore) Delete(key string) error { + if !strings.HasPrefix(key, s.prefix) { + key = s.prefix + key + } + return s.client.Del(key).Err() +} From 7709bc6e868911acc8e87823295ac8412bf15739 Mon Sep 17 00:00:00 2001 From: Hylke Visser Date: Mon, 13 Mar 2017 10:32:19 +0100 Subject: [PATCH 12/13] Use redis.Set instead of transactional Create/Update --- core/discovery/announcement/store.go | 7 +++---- core/handler/application/store.go | 8 ++------ core/handler/device/store.go | 9 ++------- core/networkserver/device/store.go | 7 ++----- 4 files changed, 9 insertions(+), 22 deletions(-) diff --git a/core/discovery/announcement/store.go b/core/discovery/announcement/store.go index 429204ab7..fbb38bcd7 100644 --- a/core/discovery/announcement/store.go +++ b/core/discovery/announcement/store.go @@ -160,14 +160,13 @@ func (s *RedisAnnouncementStore) GetForAppEUI(appEUI types.AppEUI) (*Announcemen // Set a new Announcement or update an existing one // The metadata of the announcement is ignored, as metadata should be managed with AddMetadata and RemoveMetadata func (s *RedisAnnouncementStore) Set(new *Announcement) error { - key := fmt.Sprintf("%s:%s", new.ServiceName, new.ID) now := time.Now() new.UpdatedAt = now - err := s.store.Update(key, *new) - if errors.GetErrType(err) == errors.NotFound { + key := fmt.Sprintf("%s:%s", new.ServiceName, new.ID) + if new.old == nil { new.CreatedAt = now - err = s.store.Create(key, *new) } + err := s.store.Set(key, *new) if err != nil { return err } diff --git a/core/handler/application/store.go b/core/handler/application/store.go index 34da4efa4..82bc93aaa 100644 --- a/core/handler/application/store.go +++ b/core/handler/application/store.go @@ -76,17 +76,13 @@ func (s *RedisApplicationStore) Get(appID string) (*Application, error) { func (s *RedisApplicationStore) Set(new *Application, properties ...string) (err error) { now := time.Now() new.UpdatedAt = now - - if new.old != nil { - err = s.store.Update(new.AppID, *new, properties...) - } else { + if new.old == nil { new.CreatedAt = now - err = s.store.Create(new.AppID, *new, properties...) } + err = s.store.Set(new.AppID, *new, properties...) if err != nil { return } - return nil } diff --git a/core/handler/device/store.go b/core/handler/device/store.go index e1e49e900..5034c234d 100644 --- a/core/handler/device/store.go +++ b/core/handler/device/store.go @@ -104,21 +104,16 @@ func (s *RedisDeviceStore) DownlinkQueue(appID, devID string) (DownlinkQueue, er // Set a new Device or update an existing one func (s *RedisDeviceStore) Set(new *Device, properties ...string) (err error) { - now := time.Now() new.UpdatedAt = now - key := fmt.Sprintf("%s:%s", new.AppID, new.DevID) - if new.old != nil { - err = s.store.Update(key, *new, properties...) - } else { + if new.old == nil { new.CreatedAt = now - err = s.store.Create(key, *new, properties...) } + err = s.store.Set(key, *new, properties...) if err != nil { return } - return nil } diff --git a/core/networkserver/device/store.go b/core/networkserver/device/store.go index 272ce5f56..f6d1f9a41 100644 --- a/core/networkserver/device/store.go +++ b/core/networkserver/device/store.go @@ -126,14 +126,11 @@ func (s *RedisDeviceStore) Set(new *Device, properties ...string) (err error) { now := time.Now() new.UpdatedAt = now - key := fmt.Sprintf("%s:%s", new.AppEUI, new.DevEUI) - if new.old != nil { - err = s.store.Update(key, *new, properties...) - } else { + if new.old == nil { new.CreatedAt = now - err = s.store.Create(key, *new, properties...) } + err = s.store.Set(key, *new, properties...) if err != nil { return } From 4c04eb8a4212753371b3889f9206f34226bb5ae0 Mon Sep 17 00:00:00 2001 From: Hylke Visser Date: Tue, 14 Mar 2017 10:46:36 +0100 Subject: [PATCH 13/13] Add extra test cases for Redis Map Store --- core/storage/redis_map_store_test.go | 31 +++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/core/storage/redis_map_store_test.go b/core/storage/redis_map_store_test.go index ca5a10b4b..538fe5e2c 100644 --- a/core/storage/redis_map_store_test.go +++ b/core/storage/redis_map_store_test.go @@ -20,6 +20,11 @@ type testRedisStruct struct { UpdatedAt Time `redis:"updated_at,omitempty"` Empty *map[string]string `redis:"empty"` NotEmpty *map[string]string `redis:"not_empty"` + changed []string +} + +func (a testRedisStruct) ChangedFields() (changed []string) { + return a.changed } func TestRedisMapStore(t *testing.T) { @@ -34,6 +39,7 @@ func TestRedisMapStore(t *testing.T) { Name: "My Name", UpdatedAt: Time{now}, NotEmpty: ¬Empty, + changed: []string{"Name", "UpdatedAt", "NotEmpty"}, } s.SetBase(testRedisStructVal, "") @@ -90,7 +96,8 @@ func TestRedisMapStore(t *testing.T) { c.Del("test-redis-map-store:" + name).Result() }() s.Create(name, testRedisStruct{ - Name: name, + Name: name, + changed: []string{"Name"}, }) } } @@ -139,6 +146,28 @@ func TestRedisMapStore(t *testing.T) { a.So(err, ShouldNotBeNil) } + // Create/Update/Set using ChangedFields + { + err := s.Create("test", &testRedisStruct{}) + a.So(err, ShouldBeNil) // Create not executed, so we don't know it's already there + exists, _ := c.Exists("test-redis-map-store:new").Result() + a.So(exists, ShouldBeFalse) + + err = s.Update("not-there", &testRedisStruct{}) + a.So(err, ShouldBeNil) // Update not executed, so we don't know it's not there + exists, _ = c.Exists("test-redis-map-store:new").Result() + a.So(exists, ShouldBeFalse) + + err = s.Set("test", &testRedisStruct{ + Name: "Not-changed Name", + }) + a.So(err, ShouldBeNil) + + name, err := c.HGet("test-redis-map-store:test", "name").Result() + a.So(err, ShouldBeNil) + a.So(name, ShouldEqual, "My Name") + } + // Set { err := s.Set("test", &testRedisStruct{