Merge pull request #591 from grycap/stats

Stats  #291

app/__init__.py

@@ -20,6 +20,7 @@
 # under the License.
 """Main Flask App file."""
 
+import datetime
 import yaml
 import io
 import os
@@ -1618,6 +1619,105 @@ def get_cred_id():
         else:
             return session['userid']
 
+    @app.route('/stats')
+    @authorized_with_valid_token
+    def show_stats():
+        init_date = request.args.get('init_date')
+        end_date = request.args.get('end_date')
+        active = request.args.get('active')
+        today = datetime.datetime.today().date()
+
+        if not end_date:
+            end_date = str(today)
+
+        if not init_date:
+            init_date = str(today - datetime.timedelta(days=180))
+
+        access_token = oidc_blueprint.session.token['access_token']
+
+        auth_data = utils.getIMUserAuthData(access_token, cred, get_cred_id())
+        fedcloud_sites = None
+        inf_actives = []
+        # Add an element in the first date
+        infs = [0]
+        vms = [0]
+        cpus = [0]
+        mems = [0]
+        labels = ["%s 00:00:00" % init_date]
+        cloud_hosts = []
+        clouds = [""]
+        site_name = None
+        try:
+            for inf_stat in sorted(im.get_stats(auth_data, init_date, end_date),
+                                   key=lambda stat: stat['creation_date']):
+                if inf_stat['cloud_host']:
+                    # only load this data if a EGI Cloud site appears
+                    if fedcloud_sites is None:
+                        fedcloud_sites = {}
+                        for site in list(utils.getCachedSiteList().values()):
+                            site_host = urlparse(site['url'])[1].split(":")[0]
+                            fedcloud_sites[site_host] = site["name"]
+
+                    site_name = inf_stat['cloud_host']
+                    if site_name in fedcloud_sites:
+                        site_name = fedcloud_sites[site_name]
+                elif inf_stat['cloud_type']:
+                    site_name = inf_stat['cloud_type']
+
+                if site_name not in cloud_hosts:
+                    cloud_hosts.append(site_name)
+
+                if active:
+                    curr_date = datetime.datetime.strptime(inf_stat['creation_date'], "%Y-%m-%d %H:%M:%S")
+                    inf_list = list(inf_actives)
+                    for inf in inf_list:
+                        del_time = datetime.datetime.strptime(inf[4], "%Y-%m-%d %H:%M:%S")
+                        if del_time <= curr_date:
+                            infs.append(-1)
+                            vms.append(inf[0] * -1)
+                            mems.append(inf[1] * -1.0)
+                            cpus.append(inf[2] * -1)
+                            clouds.append(inf[3])
+                            labels.append(inf[4])
+                            inf_actives.remove(inf)
+
+                    inf_actives.append((inf_stat['vm_count'], (inf_stat['memory_size'] / 1024),
+                                        inf_stat['cpu_count'], site_name, "%s 12:00:00" % inf_stat['last_date']))
+
+                infs.append(1)
+                vms.append(inf_stat['vm_count'])
+                mems.append((inf_stat['memory_size'] / 1024))
+                cpus.append(inf_stat['cpu_count'])
+                labels.append(inf_stat['creation_date'])
+                clouds.append(site_name)
+
+            if active:
+                curr_date = datetime.datetime.strptime("%s 23:59:59" % end_date, "%Y-%m-%d %H:%M:%S")
+                for inf in inf_actives:
+                    del_time = datetime.datetime.strptime(inf[4], "%Y-%m-%d %H:%M:%S")
+                    if del_time <= curr_date:
+                        infs.append(-1)
+                        vms.append(inf[0])
+                        mems.append(inf[1])
+                        cpus.append(inf[2])
+                        clouds.append(inf[3])
+                        labels.append(inf[4])
+
+            # Add an element in the last date
+            infs.append(0)
+            vms.append(0)
+            mems.append(0)
+            cpus.append(0)
+            labels.append("%s 23:59:59" % end_date)
+            clouds.append("")
+
+        except Exception as ex:
+            flash("Error Getting Stats: %s." % ex, 'error')
+
+        return render_template('stats.html', infs=infs, vms=vms, cpus=cpus, mems=mems, labels=labels,
+                               today=str(today), init_date=init_date or "", end_date=end_date or "",
+                               cloud_hosts=cloud_hosts, clouds=clouds)
+
     return app
 
 

app/im.py

@@ -25,7 +25,7 @@
 
 class InfrastructureManager():
 
-    def __init__(self, im_url, timeout=60):
+    def __init__(self, im_url, timeout=120):
         self.im_url = im_url
         self.timeout = timeout
 
@@ -160,6 +160,22 @@ def change_user(self, infid, token, overwrite, auth_data):
 
         return requests.post(url, headers=headers, timeout=self.timeout, data=new_auth)
 
+    def get_stats(self, auth_data, init_date=None, end_date=None):
+        headers = {"Authorization": auth_data}
+        url = "%s/stats" % self.im_url
+        if init_date:
+            url += "?init_date=%s" % init_date
+        if end_date:
+            if init_date:
+                url += "&"
+            else:
+                url += "?"
+            url += "end_date=%s" % end_date
+        response = requests.get(url, headers=headers, timeout=self.timeout)
+        response.raise_for_status()
+        stats = response.json()
+        return stats['stats']
+
     def remove_resources(self, infid, vm_list, auth_data):
         headers = {"Authorization": auth_data}
         url = "%s/infrastructures/%s/vms/%s" % (self.im_url, infid, vm_list)

app/settings.py

@@ -43,7 +43,7 @@ def __init__(self, config):
         self.static_sites_url = config.get('STATIC_SITES_URL', "")
         self.appdb_cache_timeout = config.get('APPDB_CACHE_TIMEOUT', 3600)
         self.debug_oidc_token = config.get('DEBUG_OIDC_TOKEN', None)
-        self.imTimeout = config.get('IM_TIMEOUT', 60)
+        self.imTimeout = config.get('IM_TIMEOUT', 120)
         self.checkToscaChangesTime = config.get('CHECK_TOSCA_CHANGES_TIME', 120)
         self.vault_url = config.get('VAULT_URL', None)
         self.im_auth = config.get('IM_AUTH', None)

app/templates/advanced_config.html

@@ -29,15 +29,17 @@ <h5>Cloud Provider:</h5>
                  borderWidth: 0
                }
            },
-           legend: {
-               display: false
-           },
            responsive: false,
-           title: {
-               display: true,
-               text: 'VCPUs',
-               padding: 0
-           }
+           plugins: {
+                legend: {
+                    display: false
+                },
+                title: {
+                    display: true,
+                    text: 'VCPUs',
+                    padding: 0
+                }   
+            }
        }
    };
 
@@ -47,10 +49,10 @@ <h5>Cloud Provider:</h5>
        config[resource_types[i]] = JSON.parse(JSON.stringify(config["gen"]));;    
    }
 
-   config["ram"].options.title.text = "RAM (GB)"
-   config["instances"].options.title.text = "Instances"
-   config["floating_ips"].options.title.text = "Float IPs"
-   config["security_groups"].options.title.text = "SGs"
+   config["ram"].options.plugins.title.text = "RAM (GB)"
+   config["instances"].options.plugins.title.text = "Instances"
+   config["floating_ips"].options.plugins.title.text = "Float IPs"
+   config["security_groups"].options.plugins.title.text = "SGs"
 
    window.onload = function() {
        window.myPies = {};

app/templates/base.html

@@ -137,6 +137,7 @@
             <a class="nav-link dropdown-toggle" data-bs-toggle="dropdown" href="#">Advanced</a>
             <div class="dropdown-menu">
               <a class="dropdown-item" href="{{ url_for("show_settings") }}">Settings</a>
+              <a class="dropdown-item" href="{{ url_for("show_stats") }}">Stats</a>
             </div>
           </li>
           {% if session["external_links"]|length > 0 %}
@@ -183,4 +184,4 @@
   body {
     margin-bottom: 70px
   }
-</style>
+</style>

app/templates/ssh_keys.html

@@ -120,4 +120,3 @@ <h5 class="modal-title">Confirm SSH Key deletion</h5>
 
 </script>
 {% endblock %}
-