From ebea2b7142c1d9989cb2dd2dac03515dd8cdffe5 Mon Sep 17 00:00:00 2001 From: torzdf <36920800+torzdf@users.noreply.github.com> Date: Thu, 27 Jun 2019 18:27:47 +0100 Subject: [PATCH] Analysis Fixups - Move stats calculations to LongRunningTasks - Fix divide by zero error on rate calculations --- CODE_OF_CONDUCT.md | 76 ++++++++++++++++ lib/gui/display_analysis.py | 169 ++++++++++++++++++++++-------------- lib/gui/stats.py | 8 +- lib/gui/utils.py | 10 +-- 4 files changed, 189 insertions(+), 74 deletions(-) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..6cc7a89219 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at deefakesrepo@gmail.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/lib/gui/display_analysis.py b/lib/gui/display_analysis.py index 11cfaee035..843dc37c8e 100644 --- a/lib/gui/display_analysis.py +++ b/lib/gui/display_analysis.py @@ -11,7 +11,7 @@ from .display_page import DisplayPage from .stats import Calculations, Session from .tooltip import Tooltip -from .utils import ControlBuilder, FileHandler, get_config, get_images +from .utils import ControlBuilder, FileHandler, get_config, get_images, LongRunningTask logger = logging.getLogger(__name__) # pylint: disable=invalid-name @@ -27,6 +27,7 @@ def __init__(self, parent, tabname, helptext): self.session = None self.add_options() self.add_main_frame() + self.thread = None # Thread for compiling stats data in background logger.debug("Initialized: %s", self.__class__.__name__) def set_vars(self): @@ -60,10 +61,8 @@ def reset_session_info(self): def load_session(self): """ Load previously saved sessions """ logger.debug("Loading session") - get_config().set_cursor_busy() fullpath = FileHandler("filename", "state").retfile if not fullpath: - get_config().set_cursor_default() return self.clear_session() logger.debug("state_file: '%s'", fullpath) @@ -71,7 +70,6 @@ def load_session(self): logger.debug("model_dir: '%s'", model_dir) model_name = self.get_model_name(model_dir, state_file) if not model_name: - get_config().set_cursor_default() return self.session = Session(model_dir=model_dir, model_name=model_name) self.session.initialize_session(is_training=False) @@ -79,7 +77,6 @@ def load_session(self): if len(msg) > 70: msg = "...{}".format(msg[-70:]) self.set_session_summary(msg) - get_config().set_cursor_default() @staticmethod def get_model_name(model_dir, state_file): @@ -96,28 +93,40 @@ def get_model_name(model_dir, state_file): def reset_session(self): """ Reset currently training sessions """ logger.debug("Reset current training session") - get_config().set_cursor_busy() self.clear_session() session = get_config().session if not session.initialized: logger.debug("Training not running") print("Training not running") - get_config().set_cursor_default() return msg = "Currently running training session" self.session = session # Reload the state file to get approx currently training iterations self.session.load_state_file() self.set_session_summary(msg) - get_config().set_cursor_default() def set_session_summary(self, message): """ Set the summary data and info message """ - logger.debug("Setting session summary. (message: '%s')", message) - self.summary = self.session.full_summary - self.set_info("Session: {}".format(message)) - self.stats.session = self.session - self.stats.tree_insert_data(self.summary) + if self.thread is None: + logger.debug("Setting session summary. (message: '%s')", message) + self.thread = LongRunningTask(target=self.summarise_data, args=(self.session, )) + self.thread.start() + self.after(1000, lambda msg=message: self.set_session_summary(msg)) + elif not self.thread.complete.is_set(): + logger.debug("Data not yet available") + self.after(1000, lambda msg=message: self.set_session_summary(msg)) + else: + logger.debug("Retrieving data from thread") + self.summary = self.thread.get_result() + self.thread = None + self.set_info("Session: {}".format(message)) + self.stats.session = self.session + self.stats.tree_insert_data(self.summary) + + @staticmethod + def summarise_data(session): + """ Summarise data in a LongRunningThread as it can take a while """ + return session.full_summary def clear_session(self): """ Clear sessions stats """ @@ -130,16 +139,13 @@ def clear_session(self): def save_session(self): """ Save sessions stats to csv """ logger.debug("Saving session") - get_config().set_cursor_busy() if not self.summary: logger.debug("No summary data loaded. Nothing to save") print("No summary data loaded. Nothing to save") - get_config().set_cursor_default() return savefile = FileHandler("save", "csv").retfile if not savefile: logger.debug("No save file. Returning") - get_config().set_cursor_default() return write_dicts = [val for val in self.summary.values()] @@ -151,7 +157,6 @@ def save_session(self): csvout.writeheader() for row in write_dicts: csvout.writerow(row) - get_config().set_cursor_default() class Options(): @@ -202,6 +207,7 @@ def __init__(self, parent, selected_id, helptext): super().__init__(parent) self.pack(side=tk.TOP, padx=5, pady=5, fill=tk.BOTH, expand=True) self.session = None # set when loading or clearing from parent + self.thread = None # Thread for loading data popup self.selected_id = selected_id self.popup_positions = list() @@ -318,8 +324,6 @@ def data_popup(self, datapoints): If there are fewer data points than this, switch the default to smoothed """ - get_config().set_cursor_busy() - get_config().root.update_idletasks() logger.debug("Popping up data window") scaling_factor = get_config().scaling_factor toplevel = SessionPopUp(self.session.modeldir, @@ -339,7 +343,6 @@ def data_popup(self, datapoints): str(position[0]), str(position[1]))) toplevel.update() - get_config().set_cursor_default() def data_popup_title(self): """ Set the data popup title """ @@ -386,17 +389,18 @@ def __init__(self, model_dir, model_name, session_id, datapoints): "datapoints: %s)", self.__class__.__name__, model_dir, model_name, session_id, datapoints) super().__init__() - + self.thread = None # Thread for loading data in a background task self.default_avg = 500 self.default_view = "avg" if datapoints > self.default_avg * 2 else "smoothed" self.session_id = session_id self.session = Session(model_dir=model_dir, model_name=model_name) self.initialize_session() + self.graph_frame = None self.graph = None self.display_data = None - self.vars = dict() + self.vars = {"status": tk.StringVar()} self.graph_initialised = False self.build() logger.debug("Initialized: %s", self.__class__.__name__) @@ -418,13 +422,20 @@ def initialize_session(self): def build(self): """ Build the popup window """ logger.debug("Building popup") - optsframe, graphframe = self.layout_frames() - + optsframe = self.layout_frames() + self.set_callback() self.opts_build(optsframe) self.compile_display_data() - self.graph_build(graphframe) logger.debug("Built popup") + def set_callback(self): + """ Set a tk boolean var to callback when graph is ready to build """ + logger.debug("Setting tk graph build variable") + var = tk.BooleanVar() + var.set(False) + var.trace("w", self.graph_build) + self.vars["buildgraph"] = var + def layout_frames(self): """ Top level container frames """ logger.debug("Layout frames") @@ -434,11 +445,11 @@ def layout_frames(self): sep = ttk.Frame(self, width=2, relief=tk.RIDGE) sep.pack(fill=tk.Y, side=tk.LEFT) - rightframe = ttk.Frame(self) - rightframe.pack(side=tk.RIGHT, fill=tk.BOTH, pady=5, expand=True) + self.graph_frame = ttk.Frame(self) + self.graph_frame.pack(side=tk.RIGHT, fill=tk.BOTH, pady=5, expand=True) logger.debug("Laid out frames") - return leftframe, rightframe + return leftframe def opts_build(self, frame): """ Build Options into the options frame """ @@ -581,6 +592,12 @@ def opts_buttons(self, frame): btnframe = ttk.Frame(frame) btnframe.pack(fill=tk.X, pady=5, padx=5, side=tk.BOTTOM) + lblstatus = ttk.Label(btnframe, + width=40, + textvariable=self.vars["status"], + anchor=tk.W) + lblstatus.pack(side=tk.LEFT, anchor=tk.W, fill=tk.X, expand=True) + for btntype in ("reset", "save"): cmd = getattr(self, "optbtn_{}".format(btntype)) btn = ttk.Button(btnframe, @@ -594,11 +611,9 @@ def opts_buttons(self, frame): def optbtn_save(self): """ Action for save button press """ logger.debug("Saving File") - self.config(cursor="watch") savefile = FileHandler("save", "csv").retfile if not savefile: logger.debug("Save Cancelled") - self.config(cursor="") return logger.debug("Saving to: %s", savefile) save_data = self.display_data.stats @@ -608,34 +623,26 @@ def optbtn_save(self): csvout = csv.writer(outfile, delimiter=",") csvout.writerow(fieldnames) csvout.writerows(zip(*[save_data[key] for key in fieldnames])) - self.config(cursor="") def optbtn_reset(self, *args): # pylint: disable=unused-argument """ Action for reset button press and checkbox changes""" logger.debug("Refreshing Graph") if not self.graph_initialised: return - self.config(cursor="watch") - self.update_idletasks() valid = self.compile_display_data() if not valid: logger.debug("Invalid data") - self.config(cursor="") return self.graph.refresh(self.display_data, self.vars["display"].get(), self.vars["scale"].get()) - self.config(cursor="") logger.debug("Refreshed Graph") def graph_scale(self, *args): # pylint: disable=unused-argument """ Action for changing graph scale """ if not self.graph_initialised: return - self.config(cursor="watch") - self.update_idletasks() self.graph.set_yscale_type(self.vars["scale"].get()) - self.config(cursor="") @staticmethod def set_help(control): @@ -669,30 +676,48 @@ def set_help(control): def compile_display_data(self): """ Compile the data to be displayed """ - logger.debug("Compiling Display Data") + if self.thread is None: + logger.debug("Compiling Display Data in background thread") + loss_keys = [key for key, val in self.vars["loss_keys"].items() + if val.get()] + logger.debug("Selected loss_keys: %s", loss_keys) - loss_keys = [key for key, val in self.vars["loss_keys"].items() - if val.get()] - logger.debug("Selected loss_keys: %s", loss_keys) + selections = self.selections_to_list() - selections = self.selections_to_list() + if not self.check_valid_selection(loss_keys, selections): + logger.warning("No data to display. Not refreshing") + return False + self.vars["status"].set("Loading Data...") + kwargs = dict(session=self.session, + display=self.vars["display"].get(), + loss_keys=loss_keys, + selections=selections, + avg_samples=self.vars["avgiterations"].get(), + smooth_amount=self.vars["smoothamount"].get(), + flatten_outliers=self.vars["outliers"].get(), + is_totals=self.is_totals) + self.thread = LongRunningTask(target=self.get_display_data, kwargs=kwargs, widget=self) + self.thread.start() + self.after(1000, self.compile_display_data) + elif not self.thread.complete.is_set(): + logger.debug("Popup Data not yet available") + self.after(1000, self.compile_display_data) + else: + logger.debug("Getting Popup from background Thread") + self.display_data = self.thread.get_result() + self.thread = None + if not self.check_valid_data(): + logger.warning("No valid data to display. Not refreshing") + self.vars["status"].set("") + return False + logger.debug("Compiled Display Data") + self.vars["buildgraph"].set(True) + return True - if not self.check_valid_selection(loss_keys, selections): - logger.warning("No data to display. Not refreshing") - return False - self.display_data = Calculations(session=self.session, - display=self.vars["display"].get(), - loss_keys=loss_keys, - selections=selections, - avg_samples=self.vars["avgiterations"].get(), - smooth_amount=self.vars["smoothamount"].get(), - flatten_outliers=self.vars["outliers"].get(), - is_totals=self.is_totals) - if not self.check_valid_data(): - logger.warning("No valid data to display. Not refreshing") - return False - logger.debug("Compiled Display Data") - return True + @staticmethod + def get_display_data(**kwargs): + """ Get the display data in a LongRunningTask """ + return Calculations(**kwargs) def check_valid_selection(self, loss_keys, selections): """ Check that there will be data to display """ @@ -726,14 +751,24 @@ def selections_to_list(self): logger.debug("Compiling selections to list: %s", selections) return selections - def graph_build(self, frame): + def graph_build(self, *args): # pylint:disable=unused-argument """ Build the graph in the top right paned window """ + if not self.vars["buildgraph"].get(): + return + self.vars["status"].set("Loading Data...") logger.debug("Building Graph") - self.graph = SessionGraph(frame, - self.display_data, - self.vars["display"].get(), - self.vars["scale"].get()) - self.graph.pack(expand=True, fill=tk.BOTH) - self.graph.build() - self.graph_initialised = True + if self.graph is None: + self.graph = SessionGraph(self.graph_frame, + self.display_data, + self.vars["display"].get(), + self.vars["scale"].get()) + self.graph.pack(expand=True, fill=tk.BOTH) + self.graph.build() + self.graph_initialised = True + else: + self.graph.refresh(self.display_data, + self.vars["display"].get(), + self.vars["scale"].get()) + self.vars["status"].set("") + self.vars["buildgraph"].set(False) logger.debug("Built Graph") diff --git a/lib/gui/stats.py b/lib/gui/stats.py index fa777edb0a..fae76b70ea 100644 --- a/lib/gui/stats.py +++ b/lib/gui/stats.py @@ -88,7 +88,8 @@ def get_timestamps(self, session=None): continue for logfile in sides.values(): timestamps = [event.wall_time - for event in tf.train.summary_iterator(logfile)] + for event in tf.train.summary_iterator(logfile) + if event.summary.value] logger.debug("Total timestamps for session %s: %s", sess, len(timestamps)) all_timestamps[sess] = timestamps break # break after first file read @@ -276,7 +277,7 @@ def sessions_stats(self): "start": ts_data["start_time"], "end": ts_data["end_time"], "elapsed": elapsed, - "rate": (batchsize * iterations) / elapsed, + "rate": (batchsize * iterations) / elapsed if elapsed != 0 else 0, "batch": batchsize, "iterations": iterations}) compiled = sorted(compiled, key=lambda k: k["session"]) @@ -439,6 +440,9 @@ def calc_rate_total(self): batchsize = batchsizes[sess_id] timestamps = total_timestamps[sess_id] iterations = range(len(timestamps) - 1) + print("===========\n") + print(timestamps[:100]) + print([batchsize / (timestamps[i + 1] - timestamps[i]) for i in iterations][:100]) rate.extend([batchsize / (timestamps[i + 1] - timestamps[i]) for i in iterations]) logger.debug("Calculated totals rate: Item_count: %s", len(rate)) return rate diff --git a/lib/gui/utils.py b/lib/gui/utils.py index 49471b3493..60e3b16d4b 100644 --- a/lib/gui/utils.py +++ b/lib/gui/utils.py @@ -568,14 +568,14 @@ def tools_command_tabs(self): def set_cursor_busy(self, widget=None): """ Set the root or widget cursor to busy """ - logger.trace("Setting cursor to busy") + logger.debug("Setting cursor to busy. widget: %s", widget) widget = self.root if widget is None else widget widget.config(cursor="watch") widget.update_idletasks() def set_cursor_default(self, widget=None): """ Set the root or widget cursor to default """ - logger.trace("Setting cursor to default") + logger.debug("Setting cursor to default. widget: %s", widget) widget = self.root if widget is None else widget widget.config(cursor="") widget.update_idletasks() @@ -940,7 +940,7 @@ def __init__(self, group=None, target=None, name=None, args=(), kwargs=None, *, self.widget = widget self._config = get_config() self._config.set_cursor_busy(widget=self.widget) - self._complete = Event() + self.complete = Event() self._queue = Queue() logger.debug("Initialized %s", self.__class__.__name__,) @@ -950,7 +950,7 @@ def run(self): if self._target: retval = self._target(*self._args, **self._kwargs) self._queue.put(retval) - self._complete.set() + self.complete.set() finally: # Avoid a refcycle if the thread is running a function with # an argument that has a member that points to the thread. @@ -958,7 +958,7 @@ def run(self): def get_result(self): """ Return the result from the queue """ - if self._complete.is_set(): + if self.complete.is_set(): logger.debug("Getting result from thread") retval = self._queue.get() logger.debug("Got result from thread")