From 09cb7bdcb51848a462515841e4b6a760253ee864 Mon Sep 17 00:00:00 2001 From: reshinthadithyan Date: Sun, 4 Jul 2021 10:19:48 +0530 Subject: [PATCH 01/15] adding codexglue link --- metrics/bleu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/bleu.py b/metrics/bleu.py index 49c56ca..36b7839 100644 --- a/metrics/bleu.py +++ b/metrics/bleu.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -# Took the following from CodeXGlue Repository - https://github.com/microsoft/CodeXGLUE/blob/main/Code-Code/code-to-code-trans/evaluator/CodeBLEU/bleu.py +# The following code is taken from CodeXGlue Repository - https://github.com/microsoft/CodeXGLUE/blob/main/Code-Code/code-to-code-trans/evaluator/CodeBLEU/bleu.py """Python implementation of BLEU and smooth-BLEU. From 4b5cd33de5695c4cecf22b5d589291c063c1645d Mon Sep 17 00:00:00 2001 From: reshinthadithyan Date: Sun, 4 Jul 2021 10:36:30 +0530 Subject: [PATCH 02/15] Adding exact match accuracy --- metrics/extrinsic_eval.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/metrics/extrinsic_eval.py b/metrics/extrinsic_eval.py index 24ee15a..bd6e81a 100644 --- a/metrics/extrinsic_eval.py +++ b/metrics/extrinsic_eval.py @@ -1,5 +1,25 @@ from metrics.bleu import compute_bleu + +def compute_exact_match(references,generated)->float: + """ + Computes Exact Match Accuracy. + args: + reference: list of lists of references for each translation. Each + reference should be tokenized into a list of tokens. + translation: list of translations to score. Each translation + should be tokenized into a list of tokens. + returns: + exact_match_accuracy : Float + """ + assert(len(references[0])==len(generated),"Number of Samples should be equal in References and Synthesized Outputs..") + exact_match_count = 0.0 + for gen,ref in zip(generated, references[0]): + if gen == ref: + exact_match_count += 1 + exact_match_acc = exact_match_count/len(generated) + return exact_match_acc + def compute_metrics(references,generated) -> dict: """ Calculates various metrics and returns the calculated dict of these matrics. @@ -14,5 +34,5 @@ def compute_metrics(references,generated) -> dict: metrics_dict = {} #Update as in new metrics are added over here. metrics_dict["smoothed_bleu_4"] = compute_bleu(references,generated,smooth=True) metrics_dict["bleu_4"] = compute_bleu(references,generated,smooth=False) - + metrics_dict["exact_match_acc"] = compute_exact_match(references,generated) return metrics_dict \ No newline at end of file From 1cff3f903ead3534f1584a3ddfb95bb07d682064 Mon Sep 17 00:00:00 2001 From: reshinthadithyan Date: Sun, 4 Jul 2021 14:02:29 +0530 Subject: [PATCH 03/15] Formatting --- metrics/bleu.py | 160 ++++++++++++++++++++------------------ metrics/extrinsic_eval.py | 28 ++++--- 2 files changed, 102 insertions(+), 86 deletions(-) diff --git a/metrics/bleu.py b/metrics/bleu.py index 36b7839..17a06ce 100644 --- a/metrics/bleu.py +++ b/metrics/bleu.py @@ -28,98 +28,106 @@ import math def _get_ngrams(segment, max_order): - """Extracts all n-grams upto a given maximum order from an input segment. + """Extracts all n-grams upto a given maximum order from an input segment. - Args: - segment: text segment from which n-grams will be extracted. - max_order: maximum length in tokens of the n-grams returned by this - methods. + Args: + segment: text segment from which n-grams will be extracted. + max_order: maximum length in tokens of the n-grams returned by this + methods. - Returns: - The Counter containing all n-grams upto max_order in segment - with a count of how many times each n-gram occurred. - """ - ngram_counts = collections.Counter() - for order in range(1, max_order + 1): - for i in range(0, len(segment) - order + 1): - ngram = tuple(segment[i:i+order]) - ngram_counts[ngram] += 1 - return ngram_counts + Returns: + The Counter containing all n-grams upto max_order in segment + with a count of how many times each n-gram occurred. + """ + ngram_counts = collections.Counter() + for order in range(1, max_order + 1): + for i in range(0, len(segment) - order + 1): + ngram = tuple(segment[i : i + order]) + ngram_counts[ngram] += 1 + return ngram_counts -def compute_bleu(reference_corpus, translation_corpus, max_order=4, - smooth=True): - """Computes BLEU score of translated segments against one or more references. +def compute_bleu(reference_corpus, translation_corpus, max_order=4, smooth=True): + """Computes BLEU score of translated segments against one or more references. - Args: - reference_corpus: list of lists of references for each translation. Each - reference should be tokenized into a list of tokens. - translation_corpus: list of translations to score. Each translation - should be tokenized into a list of tokens. - max_order: Maximum n-gram order to use when computing BLEU score. - smooth: Whether or not to apply Lin et al. 2004 smoothing. + Args: + reference_corpus: list of lists of references for each translation. Each + reference should be tokenized into a list of tokens. + translation_corpus: list of translations to score. Each translation + should be tokenized into a list of tokens. + max_order: Maximum n-gram order to use when computing BLEU score. + smooth: Whether or not to apply Lin et al. 2004 smoothing. - Returns: - 3-Tuple with the BLEU score, n-gram precisions, geometric mean of n-gram - precisions and brevity penalty. - """ - matches_by_order = [0] * max_order - possible_matches_by_order = [0] * max_order - reference_length = 0 - translation_length = 0 - for (references, translation) in zip(reference_corpus, - translation_corpus): - reference_length += min(len(r) for r in references) - translation_length += len(translation) + Returns: + 3-Tuple with the BLEU score, n-gram precisions, geometric mean of n-gram + precisions and brevity penalty. + """ + matches_by_order = [0] * max_order + possible_matches_by_order = [0] * max_order + reference_length = 0 + translation_length = 0 + for (references, translation) in zip(reference_corpus, translation_corpus): + reference_length += min(len(r) for r in references) + translation_length += len(translation) - merged_ref_ngram_counts = collections.Counter() - for reference in references: - merged_ref_ngram_counts |= _get_ngrams(reference, max_order) - translation_ngram_counts = _get_ngrams(translation, max_order) - overlap = translation_ngram_counts & merged_ref_ngram_counts - for ngram in overlap: - matches_by_order[len(ngram)-1] += overlap[ngram] - for order in range(1, max_order+1): - possible_matches = len(translation) - order + 1 - if possible_matches > 0: - possible_matches_by_order[order-1] += possible_matches + merged_ref_ngram_counts = collections.Counter() + for reference in references: + merged_ref_ngram_counts |= _get_ngrams(reference, max_order) + translation_ngram_counts = _get_ngrams(translation, max_order) + overlap = translation_ngram_counts & merged_ref_ngram_counts + for ngram in overlap: + matches_by_order[len(ngram) - 1] += overlap[ngram] + for order in range(1, max_order + 1): + possible_matches = len(translation) - order + 1 + if possible_matches > 0: + possible_matches_by_order[order - 1] += possible_matches - precisions = [0] * max_order - for i in range(0, max_order): - if smooth: - precisions[i] = ((matches_by_order[i] + 1.) / - (possible_matches_by_order[i] + 1.)) + precisions = [0] * max_order + for i in range(0, max_order): + if smooth: + precisions[i] = (matches_by_order[i] + 1.0) / ( + possible_matches_by_order[i] + 1.0 + ) + else: + if possible_matches_by_order[i] > 0: + precisions[i] = ( + float(matches_by_order[i]) / possible_matches_by_order[i] + ) + else: + precisions[i] = 0.0 + + if min(precisions) > 0: + p_log_sum = sum((1.0 / max_order) * math.log(p) for p in precisions) + geo_mean = math.exp(p_log_sum) else: - if possible_matches_by_order[i] > 0: - precisions[i] = (float(matches_by_order[i]) / - possible_matches_by_order[i]) - else: - precisions[i] = 0.0 + geo_mean = 0 - if min(precisions) > 0: - p_log_sum = sum((1. / max_order) * math.log(p) for p in precisions) - geo_mean = math.exp(p_log_sum) - else: - geo_mean = 0 + ratio = float(translation_length) / reference_length - ratio = float(translation_length) / reference_length + if ratio > 1.0: + bp = 1.0 + else: + bp = math.exp(1 - 1.0 / ratio) + bleu = geo_mean * bp + bleu_score_dict = { + "bleu": bleu, + "precision": precisions, + "bp": bp, + "ratio": ratio, + "trans_len": translation_length, + "ref_len": reference_length, + } + return bleu_score_dict # (bleu, precisions, bp, ratio, translation_length, reference_length) - if ratio > 1.0: - bp = 1. - else: - bp = math.exp(1 - 1. / ratio) - bleu = geo_mean * bp - print(geo_mean) - bleu_score_dict = {"bleu":bleu,"precision":precisions,"bp":bp,"ratio":ratio,"trans_len":translation_length,"ref_len":reference_length} - return bleu_score_dict#(bleu, precisions, bp, ratio, translation_length, reference_length) def bleu_test_case(): """A simple functionality test case to evaluate BLEU""" - generated = [[["a","=","b","\n","y","=","a","+","1"]]] - reference = [["a","=","b","\n","print","a"]] - score_dict = compute_bleu(generated,reference,smooth=False) + generated = [[["a", "=", "b", "\n", "y", "=", "a", "+", "1"]]] + reference = [["a", "=", "b", "\n", "print", "a"]] + score_dict = compute_bleu(generated, reference, smooth=False) return score_dict + if __name__ == "__main__": score_dict = bleu_test_case() - print(score_dict) \ No newline at end of file + print(score_dict) diff --git a/metrics/extrinsic_eval.py b/metrics/extrinsic_eval.py index bd6e81a..c414912 100644 --- a/metrics/extrinsic_eval.py +++ b/metrics/extrinsic_eval.py @@ -1,7 +1,7 @@ from metrics.bleu import compute_bleu -def compute_exact_match(references,generated)->float: +def compute_exact_match(references, generated) -> float: """ Computes Exact Match Accuracy. args: @@ -12,15 +12,19 @@ def compute_exact_match(references,generated)->float: returns: exact_match_accuracy : Float """ - assert(len(references[0])==len(generated),"Number of Samples should be equal in References and Synthesized Outputs..") + assert ( + len(references[0]) == len(generated), + "Number of Samples should be equal in References and Synthesized Outputs..", + ) exact_match_count = 0.0 - for gen,ref in zip(generated, references[0]): + for gen, ref in zip(generated, references[0]): if gen == ref: exact_match_count += 1 - exact_match_acc = exact_match_count/len(generated) + exact_match_acc = exact_match_count / len(generated) return exact_match_acc -def compute_metrics(references,generated) -> dict: + +def compute_metrics(references, generated) -> dict: """ Calculates various metrics and returns the calculated dict of these matrics. args: @@ -31,8 +35,12 @@ def compute_metrics(references,generated) -> dict: returns: A dicitonary with different metrics intact. """ - metrics_dict = {} #Update as in new metrics are added over here. - metrics_dict["smoothed_bleu_4"] = compute_bleu(references,generated,smooth=True) - metrics_dict["bleu_4"] = compute_bleu(references,generated,smooth=False) - metrics_dict["exact_match_acc"] = compute_exact_match(references,generated) - return metrics_dict \ No newline at end of file + metrics_dict = { + "smoothed_bleu_4": None, + "blue_4": None, + "exact_match_acc": None, + } # Update as in new metrics are computed. + metrics_dict["smoothed_bleu_4"] = compute_bleu(references, generated, smooth=True) + metrics_dict["bleu_4"] = compute_bleu(references, generated, smooth=False) + metrics_dict["exact_match_acc"] = compute_exact_match(references, generated) + return metrics_dict From c3a5ae156a662eb71861632d61cc70ec96255782 Mon Sep 17 00:00:00 2001 From: ncoop57 Date: Fri, 9 Jul 2021 20:02:10 +0000 Subject: [PATCH 04/15] Commiting initial scripts for generating repos with loose licenses --- nbs/data_processing.ipynb | 401 ++++++++++++++++++++++++++++++++---- scripts/get_license_info.py | 31 +++ 2 files changed, 393 insertions(+), 39 deletions(-) create mode 100644 scripts/get_license_info.py diff --git a/nbs/data_processing.ipynb b/nbs/data_processing.ipynb index 4cd8119..45fff5c 100644 --- a/nbs/data_processing.ipynb +++ b/nbs/data_processing.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 4, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -16,25 +16,25 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 2, "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "File exists: ../data/repo_infos.csv\n" ] }, { - "output_type": "execute_result", "data": { "text/plain": [ "'../data/repo_infos.csv'" ] }, + "execution_count": 2, "metadata": {}, - "execution_count": 5 + "output_type": "execute_result" } ], "source": [ @@ -49,9 +49,18 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/nathan/nathan_venv/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3441: DtypeWarning: Columns (3) have mixed types.Specify dtype option on import or set low_memory=False.\n", + " exec(code_obj, self.user_global_ns, self.user_ns)\n" + ] + } + ], "source": [ "our_repos = pd.read_csv(data_path/\"repo_infos.csv\", parse_dates=True)\n", "eleuther_repos = pd.read_csv(\n", @@ -61,18 +70,166 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 4, "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ - "array([ 28043.1 , 70708.05, 338060.77])" + "array(['GNU General Public License v3.0', 'Other',\n", + " 'GNU General Public License v2.0', 'MIT License', 'The Unlicense',\n", + " 'GNU Lesser General Public License v3.0', 'Apache License 2.0',\n", + " 'BSD 2-Clause Simplified License',\n", + " 'BSD 3-Clause New or Revised License',\n", + " 'GNU Lesser General Public License v2.1',\n", + " 'Open Software License 3.0',\n", + " 'Do What The F*ck You Want To Public License',\n", + " 'Microsoft Public License', 'zlib License',\n", + " 'GNU Affero General Public License v3.0',\n", + " 'Mozilla Public License 2.0',\n", + " 'Creative Commons Attribution Share Alike 4.0 International',\n", + " 'Boost Software License 1.0', 'European Union Public License 1.2',\n", + " 'ISC License', 'Creative Commons Zero v1.0 Universal',\n", + " 'Eclipse Public License 2.0', 'Eclipse Public License 1.0',\n", + " 'Creative Commons Attribution 4.0 International',\n", + " 'PostgreSQL License', 'BSD Zero Clause License',\n", + " 'Microsoft Reciprocal License', 'BSD 3-Clause Clear License',\n", + " 'Artistic License 2.0', 'SIL Open Font License 1.1',\n", + " 'University of Illinois/NCSA Open Source License',\n", + " 'European Union Public License 1.1',\n", + " 'Educational Community License v2.0',\n", + " 'Universal Permissive License v1.0', 'Academic Free License v3.0',\n", + " 'BSD 4-Clause Original or Old License', 'MIT No Attribution',\n", + " 'ODC Open Database License v1.0'], dtype=object)" ] }, + "execution_count": 4, "metadata": {}, - "execution_count": 21 + "output_type": "execute_result" + } + ], + "source": [ + "our_repos.license.unique()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/nathan/nathan_venv/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3441: DtypeWarning: Columns (3) have mixed types.Specify dtype option on import or set low_memory=False.\n", + " exec(code_obj, self.user_global_ns, self.user_ns)\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAIWCAYAAABJMsq1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAC3iElEQVR4nOydd5glRdXGf2eXHJYMkpMEAckiCH4kERAUVIIEQUBBCaKoiBEFkSBBQUFRoqIkUXKSHCTssgtLZmGJkiSbQPD9/jjVO3V7OtSduTuz7Nb7PP3c29XVVae7q/tUnWiSyMjIyMjIqMOI4SYgIyMjI2PKRmYUGRkZGRmNyIwiIyMjI6MRmVFkZGRkZDQiM4qMjIyMjEZMN9wE9BrzzjuvllhiieEmIyMjI+NdhTFjxvxd0nxVx6Y6RrHEEkswevTo4SYjIyMj410FM3ui7lgWPWVkZGRkNCIzioyMjIyMRmRGkZGRkZHRiMwoMjIyMjIakRlFRkZGRkYjMqPIyMjIyGhEZhQZGRkZGY3IjCIjIyMjoxGZUWRkZGRkNGKq88wusMRBl/Yre/yILYaBkoyMjIx3N/KKIiMjIyOjEZlRZGRkZGQ0IjOKjIyMjIxGZEaRkZGRkdGIzCgyMjIyMhqRGUVGRkZGRiMyo8jIyMjIaERmFBkZGRkZjciMIiMjIyOjEZlRZGRkZGQ0IjOKjIyMjIxGZEaRkZGRkdGIzCgyMjIyMhqRGUVGRkZGRiMyo8jIyMjIaERmFBkZGRkZjciMIiMjIyOjEZlRZGRkZGQ0IjOKjIyMjIxGtDIKM1vUzK4zs/vN7D4z2z+U/8DMnjGzcWH7WHTOt8xsgpk9ZGabRuWbhbIJZnZQVL6kmd0eys8xsxlC+Yxhf0I4vkRPrz4jIyMjoxUpK4q3ga9JWgFYG9jHzFYIx46TtGrYLgMIxz4DrAhsBpxoZiPNbCTwC2BzYAVgh6idI0Nb7wVeAfYI5XsAr4Ty40K9jIyMjIwhRCujkPSspLvC/zeAB4CFG07ZCjhb0puSJgITgLXCNkHSY5LeAs4GtjIzAzYCzg/nnwFsHbV1Rvh/PrBxqJ+RkZGRMUToSkcRRD+rAbeHon3N7B4zO9XM5gplCwNPRac9HcrqyucBXpX0dqm8o61w/LVQv0zXnmY22sxGv/jii91cUkZGRkZGC5IZhZnNBvwR+Iqk14GTgKWBVYFngWMmB4EpkHSypDUlrTnffPMNFxkZGRkZUyWSGIWZTY8zibMkXQAg6XlJ70j6H/BrXLQE8AywaHT6IqGsrvwlYE4zm65U3tFWOD5HqJ+RkZGRMURIsXoy4BTgAUnHRuULRtU+Cdwb/l8EfCZYLC0JLAPcAdwJLBMsnGbAFd4XSRJwHbBNOH9X4MKorV3D/22Aa0P9jIyMjIwhwnTtVVgX+Cww3szGhbJv41ZLqwICHgf2ApB0n5mdC9yPW0ztI+kdADPbF7gSGAmcKum+0N43gbPN7EfAWJwxEX5/a2YTgJdx5pKRkZGRMYRoZRSSbgaqLI0uazjnMOCwivLLqs6T9Bh9oqu4/D/Atm00ZmRkZGRMPmTP7IyMjIyMRmRGkZGRkZHRiMwoMjIyMjIakRlFRkZGRkYjMqPIyMjIyGhEZhQZGRkZGY3IjCIjIyMjoxGZUWRkZGRkNCIzioyMjIyMRmRGkZGRkZHRiMwoMjIyMjIakRlFRkZGRkYjMqPIyMjIyGhEZhQZGRkZGY3IjCIjIyMjoxGZUWRkZGRkNCIzioyMjIyMRmRGkZGRkZHRiMwoMjIyMjIakRlFRkZGRkYjMqPIyMjIyGhEZhQZGRkZGY3IjCIjIyMjoxGZUWRkZGRkNCIzioyMjIyMRmRGkZGRkZHRiMwoMjIyMjIakRlFRkZGRkYjWhmFmS1qZteZ2f1mdp+Z7R/K5zazq83skfA7Vyg3MzvezCaY2T1mtnrU1q6h/iNmtmtUvoaZjQ/nHG9m1tRHRkZGRsbQIWVF8TbwNUkrAGsD+5jZCsBBwDWSlgGuCfsAmwPLhG1P4CTwjz5wMPBBYC3g4OjDfxLwhei8zUJ5XR8ZGRkZGUOEVkYh6VlJd4X/bwAPAAsDWwFnhGpnAFuH/1sBZ8pxGzCnmS0IbApcLellSa8AVwObhWOjJN0mScCZpbaq+sjIyMjIGCJ0paMwsyWA1YDbgQUkPRsOPQcsEP4vDDwVnfZ0KGsqf7qinIY+ynTtaWajzWz0iy++2M0lZWRkZGS0IJlRmNlswB+Br0h6PT4WVgLqMW0daOpD0smS1pS05nzzzTc5ycjIyMiY5pDEKMxsepxJnCXpglD8fBAbEX5fCOXPAItGpy8SyprKF6kob+ojIyMjI2OIkGL1ZMApwAOSjo0OXQQUlku7AhdG5bsE66e1gdeC+OhK4KNmNldQYn8UuDIce93M1g597VJqq6qPjIyMjIwhwnQJddYFPguMN7NxoezbwBHAuWa2B/AEsF04dhnwMWAC8C9gNwBJL5vZocCdod4hkl4O//cGTgdmBi4PGw19ZGRkZGQMEVoZhaSbAas5vHFFfQH71LR1KnBqRfloYKWK8peq+sjIyMjIGDpkz+yMjIyMjEZkRpGRkZGR0YjMKDIyMjIyGpEZRUZGRkZGIzKjyMjIyMhoRGYUGRkZGRmNyIwiIyMjI6MRmVFkZGRkZDQiM4qMjIyMjEZkRpGRkZGR0YjMKDIyMjIyGpEZRUZGRkZGIzKjyMjIyMhoRGYUGRkZGRmNyIwiIyMjI6MRmVFkZGRkZDQiM4qMjIyMjEZkRpGRkZGR0YjMKDIyMjIyGpEZRUZGRkZGIzKjyMjIyMhoRGYUGRkZGRmNyIwiIyMjI6MRmVFkZGRkZDQiM4qMjIyMjEZkRpGRkZGR0YjMKDIyMjIyGpEZRUZGRkZGI1oZhZmdamYvmNm9UdkPzOwZMxsXto9Fx75lZhPM7CEz2zQq3yyUTTCzg6LyJc3s9lB+jpnNEMpnDPsTwvElenbVGRkZGRnJSFlRnA5sVlF+nKRVw3YZgJmtAHwGWDGcc6KZjTSzkcAvgM2BFYAdQl2AI0Nb7wVeAfYI5XsAr4Ty40K9jIyMjIwhRiujkHQj8HJie1sBZ0t6U9JEYAKwVtgmSHpM0lvA2cBWZmbARsD54fwzgK2jts4I/88HNg71MzIyMjKGEIPRUexrZvcE0dRcoWxh4KmoztOhrK58HuBVSW+XyjvaCsdfC/X7wcz2NLPRZjb6xRdfHMQlZWRkZGSUMVBGcRKwNLAq8CxwTK8IGggknSxpTUlrzjfffMNJSkZGRsZUhwExCknPS3pH0v+AX+OiJYBngEWjqouEsrryl4A5zWy6UnlHW+H4HKF+RkZGRsYQYkCMwswWjHY/CRQWURcBnwkWS0sCywB3AHcCywQLpxlwhfdFkgRcB2wTzt8VuDBqa9fwfxvg2lA/IyMjI2MIMV1bBTP7A7ABMK+ZPQ0cDGxgZqsCAh4H9gKQdJ+ZnQvcD7wN7CPpndDOvsCVwEjgVEn3hS6+CZxtZj8CxgKnhPJTgN+a2QRcmf6ZwV5sRkZGRkb3aGUUknaoKD6loqyofxhwWEX5ZcBlFeWP0Se6isv/A2zbRl9GRkZGxuRF9szOyMjIyGhEZhQZGRkZGY3IjCIjIyMjoxGZUWRkZGRkNCIzioyMjIyMRmRGkZGRkZHRiMwoMjIyMjIakRlFRkZGRkYjMqPIyMjIyGhEZhQZGRkZGY3IjCIjIyMjoxGZUWRkZGRkNCIzioyMjIyMRmRGkZGRkZHRiMwoMjIyMjIakRlFRkZGRkYjMqPIyMjIyGhEZhQZGRkZGY3IjCIjIyMjoxGZUWRkZGRkNCIzioyMjIyMRmRGkZGRkZHRiMwoMjIyMjIakRlFRkZGRkYjMqPIyMjIyGhEZhQZGRkZGY3IjCIjIyMjoxGtjMLMTjWzF8zs3qhsbjO72sweCb9zhXIzs+PNbIKZ3WNmq0fn7BrqP2Jmu0bla5jZ+HDO8WZmTX1kZGRkZAwtUlYUpwOblcoOAq6RtAxwTdgH2BxYJmx7AieBf/SBg4EPAmsBB0cf/pOAL0TnbdbSR0ZGRkbGEKKVUUi6EXi5VLwVcEb4fwawdVR+phy3AXOa2YLApsDVkl6W9ApwNbBZODZK0m2SBJxZaquqj4yMjIyMIcRAdRQLSHo2/H8OWCD8Xxh4Kqr3dChrKn+6orypj34wsz3NbLSZjX7xxRcHcDkZGRkZGXUYtDI7rATUA1oG3IekkyWtKWnN+eabb3KSkpGRkTHNYaCM4vkgNiL8vhDKnwEWjeotEsqayhepKG/qIyMjIyNjCDFQRnERUFgu7QpcGJXvEqyf1gZeC+KjK4GPmtlcQYn9UeDKcOx1M1s7WDvtUmqrqo+MjIyMjCHEdG0VzOwPwAbAvGb2NG69dARwrpntATwBbBeqXwZ8DJgA/AvYDUDSy2Z2KHBnqHeIpEJBvjduWTUzcHnYaOgjIyMjI2MI0cooJO1Qc2jjiroC9qlp51Tg1Iry0cBKFeUvVfWRkZGRkTG0yJ7ZGRkZGRmNyIwiIyMjI6MRmVFkZGRkZDQiM4qMjIyMjEZkRpGRkZGR0YjMKDIyMjIyGpEZRUZGRkZGIzKjyMjIyMhoRGYUGRkZGRmNyIwiIyMjI6MRmVFkZGRkZDQiM4qMjIyMjEZkRpGRkZGR0YjW6LFTM5Y46NKO/ceP2GJAdTIyMjKmZuQVRUZGRkZGIzKjyMjIyMhoRGYUGRkZGRmNyIwiIyMjI6MRmVFkZGRkZDQiM4qMjIyMjEZkRpGRkZGR0YjMKDIyMjIyGpEZRUZGRkZGIzKjyMjIyMhoRGYUGRkZGRmNyIwiIyMjI6MRmVFkZGRkZDQiM4qMjIyMjEYMilGY2eNmNt7MxpnZ6FA2t5ldbWaPhN+5QrmZ2fFmNsHM7jGz1aN2dg31HzGzXaPyNUL7E8K5Nhh6MzIyMjK6Ry9WFBtKWlXSmmH/IOAaScsA14R9gM2BZcK2J3ASOGMBDgY+CKwFHFwwl1DnC9F5m/WA3oyMjIyMLjA5RE9bAWeE/2cAW0flZ8pxGzCnmS0IbApcLellSa8AVwObhWOjJN0mScCZUVsZGRkZGUOEwTIKAVeZ2Rgz2zOULSDp2fD/OWCB8H9h4Kno3KdDWVP50xXl/WBme5rZaDMb/eKLLw7mejIyMjIyShhsKtT1JD1jZvMDV5vZg/FBSTIzDbKPVkg6GTgZYM0115zs/WVkZGRMSxjUikLSM+H3BeBPuI7h+SA2Ivy+EKo/Aywanb5IKGsqX6SiPCMjIyNjCDFgRmFms5rZ7MV/4KPAvcBFQGG5tCtwYfh/EbBLsH5aG3gtiKiuBD5qZnMFJfZHgSvDsdfNbO1g7bRL1FZGRkZGxhBhMKKnBYA/BYvV6YDfS7rCzO4EzjWzPYAngO1C/cuAjwETgH8BuwFIetnMDgXuDPUOkfRy+L83cDowM3B52DIyMjIyhhADZhSSHgNWqSh/Cdi4olzAPjVtnQqcWlE+GlhpoDQOFZY46NKO/ceP2GKYKMnIyMjoPbJndkZGRkZGIzKjyMjIyMhoRGYUGRkZGRmNyIwiIyMjI6MRmVFkZGRkZDQiM4qMjIyMjEZkRpGRkZGR0YjMKDIyMjIyGpEZRUZGRkZGIzKjyMjIyMhoRGYUGRkZGRmNyIwiIyMjI6MRmVFkZGRkZDQiM4qMjIyMjEZkRpGRkZGR0YjMKDIyMjIyGpEZRUZGRkZGIzKjyMjIyMhoRGYUGRkZGRmNGHDO7Ix0lHNqQ/+82il1MjIyMoYDeUWRkZGRkdGIzCgyMjIyMhqRGUVGRkZGRiMyo8jIyMjIaERmFBkZGRkZjciMIiMjIyOjEdk89l2EsgltlflsSp2MjIyMbpAZxTSIzEwyMjK6wRTPKMxsM+BnwEjgN5KOGGaSpnr0ykEwM6SMjKkDUzSjMLORwC+ATYCngTvN7CJJ9w8vZRm9QmYmGRlTPqZoRgGsBUyQ9BiAmZ0NbAVkRjENoRe6mbxKysgYOEzScNNQCzPbBthM0ufD/meBD0rat1RvT2DPsLsc8FB0eF7g7y1dTUl1piRaUupMSbT0qs6UREtKnSmJlpQ6UxItKXWmJFp6Vafq+OKS5qusLWmK3YBtcL1Esf9Z4OddtjH63VRnSqJlaqR3arymKYmWTO+7o05KG/E2pftRPAMsGu0vEsoyMjIyMoYIUzqjuBNYxsyWNLMZgM8AFw0zTRkZGRnTFKZoZbakt81sX+BK3Dz2VEn3ddnMye+yOlMSLSl1piRaelVnSqIlpc6UREtKnSmJlpQ6UxItvaqT0sYkTNHK7IyMjIyM4ceULnrKyMjIyBhmZEaRkZGRkdGIzCgGCTOb28zmHm46MjKmBeT3bXgw1ekozGwB4MfAQpI2N7MVgHUknRKOzwFsBiwcTnkGuFLSq1EbI4C1Jd1a08diwFHAxsCrgAGjgGuBgyQ9XqJnUl+Sni+1tSxwErCApJXMbGXgE5J+FGj9FrA1MD8g4AXgQuCIEs2zAF8DFpP0BTNbBlhO0iWp191GT1RncWAZSX8xs5mB6SS9EY4tj3vPx/1cJOmBqntZ6ns3SaeZ2cXhWish6ROl82rpieqsF+qcZmbzAbNJmhiONT6j1Dot19b4cZP0clS37Vm2janWZ5A6HmquZTZJ/4j22965xjEV6F0YuL3U7maSrujmfauhdzdJp0X78wFfAJYgMuiRtHsivcn3rmncheO1z9rMpgP2AD4JLBT1dSFwiqT/Ru0YsBOwlKRDwj17D3BXahuN6Mbp4t2wAZcD2wF3h/3pgPHh/y7Ao2EQfDdsvwxlu5TaGdvQx1+B7YGRUdlI3Hz3trC/KnAb8ADwl7A9GMpWj867AQ9VMjYquzf8Xgl8E3hPdOw9oeyqEk3nAAdG584CjBvAddfSE/5/ATdbfjTsLwNcE/5/ExgHHATsHLaDirKEZ/dk+F2/aSudU0tPVOdg4GLg4bC/EHBLyjNKrPMG8HrF9gbweqgzEXgs/Ja3x1KeZSItrc+gm/HQ9JxS3rmEMf5lPJLCn4HHga2iOnelvm9d0nsrcGSg+dPFlkhvN+9S5bjr4r39Q+hnbdyHbJHw/yTgnFI7J+Fx8R4I+3Ph70VyG433MLXiu2UD7gy/8UMubvxDwJwV58xVPMyo7OgwgKyi/iMN/T9S9ImHGykfX7t4oVLobejnodL+6Ip2ihe3m+uupSe6rhlKxwtG/DAwfUU/M0T35Z6abTzw5gCedy09pTpWqnNPyjNKfY49HsOVzzKR3pRn0DoegANqtq8BL3c5ZprG+Hh8lg0+wx8N7B/XJ+19Sx5XMW01bfbqG1I57lKedfEsG2gs93VXVTvdtNG0TdF+FAPEP81sHoLowszWBl4Lx4xqkcb/wrEYe+Evxjtm9u/iXEmjgDFmdiJwBvBUqL8osCswNuzPKun2ckeSbjOzWaOiv5vZ0hG92wDPhmNPmNmBwBkK4oWwzP9c1G+Bt4LYpWhnaeDNAVx3Ez3gL91bvtKFsDwu2v4fPmt6otTmguEYwALApsArpTqGz/T6CnwZfjiwAjBTUS5pqUR6CrwlSWZWXFNx/1OeUepzjOmev0Tvk6Xjc+Ern7jOjTG9Nc9yrgRaUp5Bynj4MfAT4O2KemXdZtM7B81jaoSCuEnS42a2AXB+ECcWtKS8b8njCrjEzD4m6bKKa2ujt5t3qW7cddRpeG9fNrNtgT9K+l84PgLYtuI6/xuibRftzBdo6qaNeqRylHfLBqyOixVeC78PAyuHY7vSt2z8dtiKZePnuuhjBuBLwBX4jGU8vvzeG5gx1DkeuBRfMn8obNuHsp9HbS2FixD+hcsObwaWUN8s5UhcvPBy2B4IZXOXaNoEXzK/CJyFL+M36Pa6m+gJx48K5z8Y+vwTcFg4thkwIdyLk8N2RSjbLNQ5BViv5r7+vrR/My6XvgdYHPgBcEipTi09UZ2vA7/CRT9fwEUZ+6U8o9TnGOp+AngE+CcuUvofcF+pzufDeHkFuA74N3BtyrNMpDflGbSOB/zjukbNc3oq9Z1LGOPXAquW2psOOBN4p4v3rZtx9UZ4Nv8J/yeJCBPo7eZdqhx3Xby3S+CiqRfDPX0Y11GeAyxZamcnPGrF08Bh+Mpn227aaNqmOmU2TJpVLodz+IfUqfSZC595lBVRr5TaKJRDS0o61MwWBRaUdEcXdGxOtVKx30wmzDZGqKSE7QZhVrc2ft23Sfp7dCzputvoCbORPYCPhn6uxAM3Kjq+VqmfOyW9M4DrGSNpDTMbL+n9cVkqPVG9TeI6kq4O5a3PKPU5mtndwEbAXyStZmYbAjtL2iOqMx74AP58Vg2K3B9L+lSprcpnmUhv6zNoGw9mthzwUjyGonMXUH8Feu07F9XpN6bMbBHgbUnPVdRfV9It5fKhQsM7kPwu1Y27Up3a97ZUB0kvNdC7PD6xMlxP90DpeGsbtUjlKO+WDeeis4f/3wUuIFIed9FOpXIo4bwtu+xnf9yCw4Df4FYKH004b/XS/rq4mARcgXksHja42+tOpgeYm2jmOBme5a24mOMCYF/ccqNJb1NJDzArQRGKf8w+QYUcvwf0FvLmu/EPDJT0GPTJv8fRNxsurzp68iyHamt75wY6xhP77up9i877BK6HPLrcRq/oTRl3A33WRAYuYX/paDxtgBsJzNlNG411h3uQ9XojKIuA9fCl/Ra42V3beSeX9iuVQwnt/DChzp7lNvEZyp+AFYu+W9r4dfm6w8BeJQzsfYAbBnDdjfQA14eXaG5cvHI7cFxCP5d0Wwefec+GW2qcFj5Aa5fqtNIDjMGtSRYOdc4Dzkp9Rql1cHHFbMAJuLXJz4BbS3X+BMyJi9FuxM0ULxvss0ykN+UZnJxQp3zdje/cIMZ4Cr0p71t5XB0BXAPsHrargcN7QG/5XWodd4N4by8t7Y/DRXbvxcVOPymPq7Y2GuumVny3bPRZShwO7BiXtZy3Rmn/dtwEr2AY86W0k0jjXvFACb8/Az6ZSm9FmwWd3wf2iMu6vO5GeqL7+/niJaVkyVHTz4KDqYOvLEY1PO9aeqJ7sx9wYPg/LvUZpdYhzCDDC7srPqubp+H89fFZ5gyDfZaJ9KY8g0q9RMt1F8+g8p0b6BhPoTdlK7eDf5xHRPsj4zEzCHrL71LruBvoe1vRd9HOgQQ9yEC+I7Xt96qhKWUDLqFPgTQnMCMDMGOkRjkUHV8L+ED4vwJuIfWxAfRzGnAVrgSdBZgdGBMdfw9hiYgzq08BK1a0cwPunPdwOGcEJTPRHtEzHreguSq6/lZGMcBn+Xt8tTArnv72aeAbpTqt9OCWMevg/gYrFucN4xgdiVsmLVZsk+NZDuH1NL5zbWMqsY+evG/F+CAyBsFXo/f0kt7UcTfQZ00wKY72bwd2AO4lKKmJ/J9S2misO9yDbDIM2lnCx3SZsL8gQb4YXtC9gEOBdUvnfbeireXxpeC+wPui8oPDwx+Nz6KuBb6HixK+Uzp/44qHuln0fwRuNTJn2J+HPiutvfAl6+O41cftuHXHQ4TZR9TOe8LL8+GwvxjBASgcK3Qu8+Bij/HAufSfbdXSE/a3DS/aiWF/Kdz0DvyjfjjwW8LMMjrvxNQ60f648LsTcAwwPf2ZQC09UZ3/w5n+N6M6x7eMo90Sxtpupf1P4R+X1yg53EV19sNTUN5HnwVP+ZqanmXjmArHL8ctoZYGTse9me8gjGG6fA8Srrv2nUsZUw39XN7N+9bWTrS/A24+fDpucjsR2D7xnUy+dynjrulZt1xT2YlwBdwqboewv2TRb2obTdvUavU0Ererjt3znzSz3+CD+g48reoNkg4I59wlafXEdsbjXrIzAs8Bi0h6PdhD3y5pZTP7Ms5kHgh195d0YVVfZrYwbv4Z93Nj6OeDwMz4wH6vpOeC1cV1klZNvB9X4B+OWYEdcTO83+OhQT4iaatS/Up6Evr5I/6hvA2X/f4XZwZvFtecUidq7z783v0eN/+8wczulrRKynUPBmb2pKTFuqljZhOAj6shXEmo80ENwPIkZUyZ2Y24fHo2XBb/TdwUckvgK5I27vY9aLvuUFb5rkTH68Z4XV+G6xYWTHzfWtsp0bsgrgMDuEMlq6sGegd177qBmR1QdwhnkK0xr3rRBkzhiYsGAjPbD5+BPE+fg5GAlYG1JK0c6v0cONHMLsBnGNbQzjv0OdqsjJvzvQP8y8welfQ6gKR/m1nR5xdwmeU/zGwJ3IloCUk/i/sysyNxW/j7Qz8FvTcC/5X0r6if50I/rxROPFE7n8L9K+YP7ccOggtIOiHU21vSkeG0E8xsj1I7TfS0xclZWtKnQ9Gfzew7wLVmFsdmSqlT4Jf4aupu4MbghPV6id7GuD2hzrK4TXtHHTzBfBUM/+hhZve01YnwfBOTCHiKTme0/g3XPEt8stA2pmaXdHFo51BJZ4fyi83sh+F/63vQzXW3vHNtY+pOXPxSdlYDF2NB2vvW2o6ZLS/pwYipPB1+FzKzhSTdlUBvN9+QynEnaaOoTtN7m+z0aGbr4pKCgrkVY2ah1DaaMNUxCty0bbmaGdsMxR9JbwN7mtn38aXsbF2085aZzRI+4rFN/xz0vSgpHqfgs/rlJL1Jf8jMppfbpG8R9TMT/R/yUdTPZuO6ZzYca6MH3ErnJtzCp+wbMaOZjVDwAJV0mJk9g79gs3VRp/AFeF7SwlHZk8CGXdBT4Dyc6fymVOcy2r15u/H4HW1m5+BxiybdP0kXRHUeA643s0tLdY6N6lQ+SzO7L2FMjYxOiduEvvGf8h50c91N7wo0j6kHcOX4I+UDZlZ4Yae8byntHADsiYsxyxDuA9NGbzffkLpxF6Ppvb0L+LOkMRXX9PlS0SnAV3FLq7ivi7toox6pMqp3y4ab501Xc+x3RPqBqPzz+Ow9tZ0Za8rnBd4f/rd6nIayy6lRKuHyyn404OZ2HymV3VLVRjh2SFUfuCnd+aWyWnrC8XENx44q0xXKN6MvJk9rnahsdMLzrqUnqlOpiCTBmzelTrR/WsV2aqnOwVVbyrNMGVO4/LzuWf809T3o8rpr35WEMb4N/lGuOrZ1+E1531rbifZnqqgzUyK93XxDWhXgLe/tcsC8NccWKO1XugB000bTNtXpKMzsFPzmNM3Yms4vZHortrVjZscDZ6siHHmqx2mQ2a+C23XH/Xy5RNM5kp5poPtnuGLsz9TMZs1spFo8pNvoMbMf4b4BdXFykpBIyxG44vccPCxGQUsckruVHjP7AR624E90XtPLdedMbpjZbIGGf1Qcq3yWuFx8ivNibnvnUsZ4Yj+171uX7VTpIyeV9ZDeH9Ay7hLf29UVxGINfR2BryYvKLVTiNNa22hsfypkFAdXlUv6YVV56vl9zeiQqO6uuCxzOXwwnC1pdKm9xsEd2qjq6IwSTdvhsZ7OAc5T/xAKp9XQG8vqn8Tj5ZyDxxfq9/Db6DGzN3Cl+Fthi2WqyUikZWLNNS0V1WmlJ6WdXiBMDk7AvW3BRWL7S3o6qrMSbvFVKBH/jlu53BfVaX2WUxLa3rmUMZ7YT+v71nL+e/DV+O9wo45CXDcK+KWk5VPpDTqcs/EJ3KM1/aWM35T39jqcmZwf+ru3oq/ratrZKLWNRqQuPd5tGzDLIM/fNqUslM+NK1Svob/4ZFdcFv4oHi5gzYrzZ6Zm2VyqtzLu0/EgHk+o63uCM5wLcCXxz6kQL6TSM9jnU0HLh4d73Azymq4GdsPFQdPhUX6vLtW5Fdgw2t+Akvf2u3Vreud6Oaaa3reW83bFxWRvhN9iuwj4VDf04krjA3GdwJ240nqxgVxPIu3vwR04b8FNqlvNmHvZxrAPrslwQ9fBrRWKJDirULLPT2ynn3dkVVkoXwtXkE0ALq6pUzm4gY/jfhETw/6qeJC3uge9X3jQZdv7ZUPbRQKUlZsGAh67qkNfkkIPPgvbGfhe2F8UtwQZzDOro2UWPHbQyWF/GfrH5WmlJ6WdHo29cW1lVDh/lsu6fZbDvbW9c92M8cT+Wt+3lvM/3XK8K3rDeBro+O32vX0/viJ9q1S+AK5XKnxPVqDka9XWRuM9Ge5B1usNd0pblJoMbWG/+Lh8P+wvVnxcgM1x8cHzuANLsZ2O21vH7RyF+wRcgc8e52ygq3Jw4zOSOVro3RuPaXQfbgK3QkX7N9CQmS4qWx84Ebe+Obf80rTRQ0KwxPCCfI8Qj6rqBUmkpTb7V5f0pLSzOEHJjs8mZ6+gt7EO/sLvjMuKR4b/5Wx7fwr3ZomwfRf4U7fPMoGW1g8HDe9Bl301vnNtYyp1zJDwviW2czAeMqNj64be6L4Uq4o7gK8NYNylPOv30eckez3ufDt/qU5blsHWNpq2If+QT+6NoP2nIZgfDR8XfDa0K+7Ys2/YvohHLp2r1M5e1FgUpA5u+lKnxvSWVwuHU7J2qeinMctY2H8c/1DtQIhYWdFOIz0kBEtMfEFSaKnN/tUlPY3tkJZONaXO4rgY40Vcifln+ofnmAufeNwVtp9VjKu2jHEptDR+ONregy77anzn2sZUF2Mm5X1Laedr0fYdPE/EqV3Se3t4fgfheaoHOn5T3tu/4ibICzVcd9uYaW2jaZsa/SieMrMPEXwQ8JtTtlH+oNyLdSxMcmAr7KPvx0NGzIB7DoPPtE7DY9pMgqRfJdDzKJ5ovl+M+YD7zGxHYKR5RrcvU7JTl/SthH7aMtOBhyF4vd+Z3dFTl0krxtKStjezHQL9/zKzsiNUCi1N2b+6oaetnX3wWd3tgd5HzLPU0U0dSU/gQf5qIc9Z0GY90/YsU+idV9K5ZvatUOdtMytbmTW9B9301fbOtY5xEsZM4vuW0k6HH4WZHY3nMemG3l0kPdRCS8r4bX1vJa3T0g+0ZBlMbKMWUyOj+CI+S1sYTyhyFT7YYzR9XI7CHWcWV0hYYmaj6Itdv383xCQM7v3wWc2beKiKK4EfddNHwD54NrPlgwPbRFysENPS9mFOoed4fCUwv5kdhtuvf7fURusLkkjLD/CV2KJmdhZuTfS5Up0Ueg5uaSclnWptHTM7UNJRZnZCxXlI+rKZ/VTSV8zs4po6MYNpe5Yp9LalJ4U0JpvSV9s7lzLGUz6qKRhIO7PgoeyT6U1gEtA+7iDhvU3EAfhqdmkzuwUPILrNANqpxFRnHpsCM9sJN7NbHQ8Ktg2uQDrPzB4BllXpxoQX6kFJyww5wV3AepApL6GPtkxam+Af6xXwj8a6eJrI6wfQV0r2r0Z62toxs6PwwHm74B+JvYH7JX0npY6ZfVzSxU1mlWa2hqQxZrZ+TZ0bKmiuy7CWQu/quK5tJTyi6HzANpLuierUvgfd9NUL9GrMpLRjHjuqeL9H4vfmEEk/H9xVVNLTOn5DvV5kuGzNMjhgDEReNSVv+ICfM9qfi5J3bCiviwz7cEPb/Y7hyVp2C//no4s8tOGcqyvovbKi3uI0KxR/XNHOjwZw/xrpwQf97NH+KFyEUW5nHjzsyJa0yJUbaLmYBh1GKj24fmmOaH9OIm9dPIzJF/CQC+eH/1ZqI6VOq0k17ldRrrN/ab/xWabQEupNhzuOrkRNRr+696DL625857oY461jJuV9a2snvEvFtjAlr/JUehPGb+O4S3nWpbpN5sf7VLSzdzdtNF7LQE6akjcqknWUy2hIG4grIPuF+cWXgxeVyg7GP2YPh/2FqHDJbxrcifSmKBSr2rmrtJ9iCdNIDx5j36L9ERX9tKZ3TKRlfdwq6gn8I7UNpfALifSMSxknoXxuWkJg19Up91vzDKrqlJ9367NMoKU1JXDTe9BlX61jJuGaU8ZM6/uW0k44tnq43v2A1bq5nrCfYl3VOu5SnjXwIVpM/tv6SmmjaUuq9G7a8Eijc0X7c9Pf2mMcNWkD8RnG7bgJ2TFhuwE3f1u4oh2j2TqicXDjpnWLRfuLVwyUcbhyPe6nfE33EMXEwVcd5VzMKZYwjfTUDMjyNd8T7ssq1KR3TKElqjsS2AQ3oS3nd0iip6JObDp4Pe3pVGvrkGBSja+MLsaD7F0UbdfRn+k3PstEeltTApOQPjOxr8Z3rm1MdTFmxtH+vqW0833cTPSHYbubyHchkd4U66rGcdfFe5ti8j+ezgnTyNKYaW2jaZsaldnHAH81s/PwAbMN7s0c439yK5BP4XkOTogsP54BPmhmG+HLdvCX55qKvt6SJAshv4OcsYxPAqvhgxZJfzOz2aPj3wFuNrMiRPKH8QiXMVIUimcB10QhAXbDRQIxUixh2uh5zDwvwklhf2/cDyLG2+G+bAX8QtIpVgpnnkgLQTH5cTpl6TFS6BltZsfipqDgH484muYc8vwGnwfOlHSw9Q+z3VTnb3hSnU+U2n0Dj+gJbjXzLB7I7phSnXJfbc8yhd7iXm6Bz3ovNY+LFaP2Peiyr7Z3LmWMp4yZlPctpZ2dgFUk/Se0cwTOhIr7k0JvimVf27iDtPcWSU+Vmi+/K1cA55hZYTyzVyjrpo16pHKUd9OGizEKH4gq57Tb6TJtYE0/X6cvBeQXcFvl/Up1ihllYe8/K/1nQfPi8tQ6mepRwLfx0B2b4FY+h1XU25w+66xNK45fj8tvC1rWpiKRexM9eNz8s3E/gedxq5Cy888NtKR3TKEFX0E8jodq3pAoz3GX9MyKJ/EZTV+WtFmj4ynpVFPqVOoBBjiGa59lIi2tKYFJeA9S+kp859rGeMqYSXnfUtq5jk55/px4vLFu6L0Vn/0X43dp+jvkNo67Lt7b83HR0V14lsev43Gu4jojcCe688O2FzCymzYax2OvBvZwb8Co8Dt31VYxqI+ni7SBDf1ugi/ZjwY2qTheObiB5cPx1au2ikHQqrxMoHV1PPzHa+H3YfpSPCbTk9BPa3rHJlqiOpvGg30yjp2UdKopdZYJz+f+8LwfAx4r1Vkb1zf9Aw9i+A4lcVqP6G1MT5r6HjT1Rcs71+UYT0oJSvv71pRG9oRwvX/GzXhPx/2jnsZ1ON3QuwnOlF7EVwWPAxtMpvE5b+jjeXxC9DtgnqFsY6oxjzWzSyRtaR6xMb6oIppoTyOFhj5nBf4j6R0zWw43TbtcJbO0YLL30UDLlZKuNrOTJe1pLVEfK/qcG08FeU/Yv1nSeuZRVKuue1Tp/EoTOjP7taQv1NEDXKEWX4EqepvQQMtGkq4NIpH+xEgXTAbfhUHDzG7GdVLH4eKy3fBV0PejOqOBz+BMf03c7HRZSd/q9lkm0NOYnnSwaHvn8MCVXY/xhv6S3reG83dtqbJuN/TWmb6mjLtePWszO1fSdiWT37ivlVPaae1namEUTTCzhRXlcrCatIHdMhMzG4PLL+cCbsaXl29J2imq0/XgNrO1Jd0W7V+Py7+nw2WcL+ARR79a3UJtu9viH/w3zOy7+EzpR0qIU9/iKyBJZ0Z1P0V9esdWWszsh3JZ+Gn0hyTtnkJPqu+CpaVTTakzRtIaZjZe0vvjsqjOaElrmtk96kupOVbSalU0ViGRlsr0pPGHI+U9SOmrhsaOd67ieHmMp4yZlPettZ2BoILedXHl9T/NbGd8/P5M0hOp466Lvo/C9Sf/xvUOKwNflfQ7M1tQ0rPmWQ6r+nqirY0kIrpZvrxbN4JJWLT/IC4XnB+Xk89Dl0u50E4hn9wPODD8H1eqMwYXAyyMW42cB5zVJb1jw+/ngR+G//3kxAnt1FrC4GKK2q2ln6NL+xOosMlPpaUHz/vohDrnRP9vxT8u2wGfLrZS/dQ6I3Axxr64IcNDpTo34hZsZ+K6p69SEVG26Vkm0jKhbUynvAcpfaWMvYSxmTJmUt632naAc8PveFyc1rF1+y7RYl3VNu666Gtc+P0kblI+R+KYuWWwbRTb1Gj1VIWyNcJrki7vRbtmtg5uRVFYVows15FbROwBnCQXl4xra7e0P52ZLYi/rN14xJbbabKE+XhDO8I/fnXYDtfFFHhe1TmAU2kBwMxmxD9MS9A5mz2kpe0yPVWIY9/MIumbLfVT6uyPTwq+DByK52Aur3g+izOTfXEmsSh+jW2In2UKLU/RP2RHGSnvQUpfVSiPvbbjKWMm5X1ramf/8LtlSz+VfZf2U6yrqpASc6nf+x9+t8CTlr3W38CqEov1oI2Ok6d2lOVr15nZT6hJG9gF9sctLP4k6T4zWwqfHcdIGdxt9B6Cx5u5WdKdoZ9+SeQT2nkmmM9tAhwZPsQjACTtltBeHarMAs+hIb1jEy0RLsQ/dmPoLu5P+hvguMTMPqbm9K6tdSTdGf7+A9dPVNV5Ivz9D27Dn4r4WabQ+xhwvZk1pQROeQ9S+mqjN+V4yphJed9q25GLaEYCp0vasJuLqaD3DXPT7p2B/zOzEbg1US9Q7usSM3sQFxt9KYgD/9NlOwNtA5iKdBR1Sk38o7GrOmWdPVGuJdL1f/js9hZJR4bB/RXcwqSO3o0kVdmIV7V/QN0h4DuS5o7qzgJshpsLPhJWKe+XdFVUp3IWj2egq+vnbkmTgqo16Ra6pOVeSSvVXPfcVeUxPebxjurqXCJpwdBWSjrVlDrLAt+gT+ZfXPhGUZ2yXqCos1Tqs0yk5eCqhhSlBE55D5r6anvncKugpDGeMmZSkDj2rsHFqR0rrjrlcw2978HTqd4p6SYzWwy3ejozZdx1896G/ubGV4DvhHdnlKTnrMbgI7TzS0nztbVRc35nY1MRo2i0aFCX+XkT+1wWZwJLUPNhaDi3UtEVtXFDVLdWoVj3QYjqdMxarcUSxsyuoG8WHzvk7Iu/RFWzdWkAVmUJtJwMnCBpfMW5haVNLT01H8K4UrezykaY2d24z0fHvZM0JqrzIC5yKtd5qdtnmUjTLJL+1e15iW23WRE93nRQ3St1B/y+ldq5EHeCvRr4Z3Toj03npdKbMu4G8N5+iP7XfWYNY4zbmbSyrWuj6fxJ504tjKIbmNkCeDCuhSRtbmYr4DkjTumynZQPw6AHt5ndCtxU0U/jwK5oJ8USpnYW30U/y+Ke0gtIWsnMVgY+IelHUZ0UWu7Hw0tMxMUIVq7TC5gLa3fCnc4ONbNFgQUl3dFlnQ4Lp5q+bpf0wSGgdx1caTmbpMXMbBVgL0l7R3Va34OUvnqBxDGT+r61tdNquZdA72Sxrqrp67e4Q984+q5b6sIkfdBtKFHrPTVtdBFnqKWdMQl17sY9JtcC1ii2LvsZ16PrTrGEORkXAQ2mnxtoT++YQsviVdtkGA8pmd5S6vwADyGyIPXOnkfgDmPrMHCHxhRaUuID9SQLXo+eQcqYSXnfUtrZv+K8fmUt/bRaafXw3jwA3TvY9rKNaUWZXUZSnKEEXGxme+MhNWLF2ctRnbclndTvzO4wUIViGSmWMOsBnwuinYHO4meRdId1WlW8nUpLpH8oYvMLeFVhxE8GpGR6S6lTzFS/EZUJ92ae1E74XbNUpxvxSQotqD22T6+y4PUCKWMm5X1LaWdXPNFSjM9VlDUhxUqrV7gX9zgvZ6wcsjamGkZhZvsqPfFISvavFKR8GFIGdxv2B75tZrXKy0SkWMJs3mWbVUhJy9pEyxj66x9mC6KHz0t6vAc0xkjJ9NZYx9zq5SBJ59R1Es6/SNJxQ0BvSkrgXmXBq0U490hJbebKKWMm5X2rbcc8gN+OwJJmdlF0zuxA4/toZkeXriHFSqtXmBe438zuKPXVTWSBQbUx1egozOwuSXXWBuW6q9OS/auHdE2sKBZwHw0mhMkPsN56omjn2KhurSWM9bci6pjFm9kH8Bloh929mX0Mn13FcuKlcBHWh/Cw2hOBneMPfBMtddcS5MJ7StosKlsbD6ccp619n6TbK85dL1zXzZL+FB1LyfSWUme0pHilUHUNd0haq6XOj4GjJL0a9ucCvibpu13QMi8+Q/4IznCvwsUrL0V1Wt+DxL4a9QJmdpuktVuuuXXMpKCpHXPv5SXx4HwHRae9gTvclVcecbtPSlos2q+1rrJ6q6ei0iTz40Q9UaXhi6QbrN7qqahzQVsbTedPomFaZBSh/qDTBpqbmB2Ax67f0zwR+3KSLkk4t3hwn8KXhIUr/Q74h/erUd1ahWL0wV0O+ACe4wDcge4OSf3y71qFJYxVWxHNhutYPg+ciidfeqJ03uLAaapQzltCescqWppQfs5BJLJ6xNBGAKNLdU7EleJ/CEXb40mg9onqpKRTbUsBewTwdzxXwSRrmnj1aGbH4fb25Trxx2OsSiE9Kq67ld4UpLwHCdd9Az7L/1VBt0VGEWZ2Eh6Z4LzSNfebfTeNmW7et5Z2jlTJibCqrHT8KUmL1h0v1W2yepI6zY8vxwMTfkfSKuF5jFUIARPVWxwP8PiXcB9GysPfFAxrfpw5Xhv2N8TD/GzZ1kbKNU12RcxQbbgc8vWK7Q36J7tJShuY0GdK8pJZ8AxjJ4f9jkxY+Eet3O7o0n6K8vJGOlOCzg7cWKqzDl1mucIZ2RXl/kp1yiGuW9M7DpCW2Sru77gEeh6EflnwHoj2U9KpptSZWLGVo8deV7GVQ1zfQ3PiohRazqh4BqeW6rS+B4l93Rl+x1Y9F/xDWN7KtKSMmZT3LaWdqiyD91AfBXce4OlS/WWBayJaViZKfpS6td27sJ+S4fIqfPJY7C9IZwrj1jYa6ez2wqbUjZrUljV1xw3m/Oic0RUPuRzzv3Fw43LjpaL9JYk+YqHsroR+HqLz4zIj/eMMDSjLFR7LZkLD8Qml/X73svxyNtGCzxrL26H46uYLpXYuwENmTB+2/YE/l+pcQmQthVtPXRzTS39GUqa3tU4Px/I38aB3e4TtZkJso27obRvjKe9BYl+X46aXxTjdhpDitotrThkzKe9bbTu49eF44F90xnmaiIfgnojrzlIYfqt1VShbCbcs26XYSsevpz0vyzjaM1yWvxnlyVBrG03bVKPM7hIjzcwU7lZQuA3EkuMt8wxsRTtL0z/URFsmrK/iSt3H8KX94njSkRgpCsUzgTvM7E9hf2sGlimrA2Y2Gz7o/mJmh+GzpoIOw8NQXFs6baSZzSjpzVBvZpxxpdIye7kq8Bwuay47330RzzHw3VDvGkI2MuvztJ0deCAo8oRbHsV+AJPGQqDrf0EEQDd1UkQjKTJpuQf/Pbi4B+BQSVd2Se8IM5tL0iuh37npb7yS8h6k9LUPrhdY3syeIegFomueCWd4KwIzRW3FXtcpYyblfWtq5/c4U+uno1B3xiWQYF0VxMIb4Hk/LsMNRW7G39UCB+Di4qXN7BaCnqjUV0qGy2vM7Eo6xat/6bKNWkxNjOK89iqT0Jo2MBEHh/MWNbOz8MTunyvVaRzckq4IH5TlQ9GDxSCPcDxuNTV/+Fhvg38YJ0HSYUHe+eFQtJuksaV2ai1hrFopPhce3vzn+Ev2G2CC9QU1XBVfzn6+dF5KesdaWtSFB7KkF/D8DlU4OrGZlHSqKXVOwy22PhT2n8HHZSxDPz3UK4I7PoyvOjucPeVGA3UB+1JoSUkJnPIetPYl6THgIw16gd/i4r9N8bhlO9HfAitlzKS8b7XtyEN2vIbrAYFJ7+M+ZvYZSSuGd3Un/OMOHsr8fElvlfpJsdLaBhepjpW0W5gkdIT1lofVX59mPdENZvZtYGbz3DZ74/nX43b2NVdsF+//yYoMNlLaaMLUpMyuizsDgCIPxKDw3Iu+GdvVwG8kde1LYTXJS6Ljm+Af9RVwOeK6wOckXR+OF7PQxeWJgyoVdInK1vVwZdVpYdUxm6SJ0fFaSxjrb4Uk4CVczzE+amMp+nKJ3xc+ElX3ZXOi+1uaESdZ5TTBEhIXpbQT2pofZ8Yb0bcq+UpgQt3UKXJNjFWfUvduSatEde6U9IFSnXGSVrXEZDYptIR6K9Dnn3GtpPtLx1vfg8Tr3h9nfm8Av8YtpA5SiNtVXKuFHBxhYnCTSpZQbWMm1Gl831LaMbOF8Bn3jsD78RVGoVi/CNf3FVZ8a+Dv7ibA19VneZZi2XeHpLXM82hsGO7PA5KWj+rsg6cdeDXsz4VnHDwxqjMCX5FNSn6GP6fkj/eg20iVUU3pG25jXbtNpj4/iSefL/bnBLauqDcPHt63Kgd1ioIuRaF4MD5DeDjsL0QUj74H17p46Vo3xD/0BwAzDMPz/njTcw/Hbg6/b9Bi4NAjmlLyKF9PQt7yQdCQnBK4h30Wnt2b4ivfFYn0C/Tljb8Rl9nPS0nmn9hP0vvWcP6euPHAw3gSn5WBidHx66hOr/oRPF3qlRXHZo3fzdKxEwONX8SjPY/FLQTjOuMqzhtb0Uec/3okLvqKx3bjGG9qI+neTY6BM6VuDCJxSU17KQ+5cXCTqKCjXaE4Dp8pxO2UrX9aLWEarvV2XK4OLnL6O/C10OZvQnnyh3kwtPRwPBTJb07AZ80dW2qdqL2P0j+P8oalOqvTP1f4KuFYndVNsaXQe0n4nUiUt5tIIUvCe9DldRfn/Az4ZMV4/nx4vusHWl4AvjiAMVP7vqW0gzur3gCsGZ3/WPT/wYaxMpHow0qCdVXp/CUo5YSPnkH8bo8ksnALZbfh0oFifzbc9LWbsT6oNqYaHYV1elr2g9yBbf+wO5DEJVUYUVFWvqcHK5IVSno1iHn+HIpSFHQpCsW3JMnMinaqwpSvrLDEDe28YmarVdSrwsyS/hb+74x/1I8JS9pxob31wm9ZGV2FWlrCte2BM9mFQpVn8PwUp0j6r9WHhC7a+4TVhyIvUIjvRlc10UWdos+rgpihEI3sr/6ikfvwD+YkmTR946jKIz3uqxi/tbQo2M1LWrJcwcwWDn9T3oPk6wbGmNlVuMXet8xsdiJjC0m/CX9voNOLutsxU/u+JbazILAtcIx5mPBz6cwhMSJWhBcIyvj/qtPfZ3NJ346u4xVz59PvRud9Ehf5vSZ3+JvTzLaW9OeonStp1xPNJOkfUV//CCLrmMYN6RMJ36sg2u6mjUZ0w5Wm5A2fxd2FO/78H/4yTtoSzu9aTIM7oR2LixiWDv9PL9Xpt1IhMkvDZZ/lWegGpfopJqBfB36Fz9i+APwV2K9U525grmh/bhJN5Eo03wVs2nSNFeeX0zvW0oJbbpyEf3AXCdvaoeycUGf9pi3UKWbVEyu2RtEHaelUyylg+9mll8uotuEftJltIr2N6UlT34OK6x6Br5TmDPvzEM2e8VDypxBMZnGZ/x4DGDOt71vqdYcx9TWcET6ArxC+S39T6iVwvcX3Suc3+rqEsnEV/Y4t7Rsumjo/bHsRiYiKZ0IUOBLXm/w1/F8YX+3fEO7HseH/HcDCKW0kja/BDtApZcOXbJvhIo2xuAxyxS7Of2oAfc6KRwMdHbbDgVm7Hdw06DDC8fmBs/El+/O4BdL8FfU2wSOTHk21rHUX3Prk0HB/HgQ+W6pT6UiEixXODb8TgenD8QWpcBpsu79NtBD0LDXt9DuGm3SujCsme6IvIe2jWjgLzoQzurtxEUQhKlqCIM7Ave/XwD9Kq9EXOXYDKkQeuKPjsbj10tY9ord1jCfWKX/A/69qi44PKFpzxZhpfd8Gck1hzH8//N8XeBIXrb4EPEFpwhXqNfq6hDptk8SRVc++4pwPAI/iqQZuxiPXrhGO/Qk3jql61y9MaSPpfejm5Xm3bLjd9OfwWfq+iee0vmgDpKVxcONWULOG/zuHj8Pik/HerBBehn2BFSqOVzoS4TOfz+B+H/FMZTWi1UU397eOFlyeui1ualmUjcAtVW4vtbEFHon2+kD7k7hYIK5zDfCxUtnJLfQmf1TxFd5EXGQYr2DuLsYfrmS/DpeZXxdtF+HZ1uJ2T8StwHYL2xV4XubB0pvM/LrpCzeiKLarcf3LtdHxVu/jgdKSOOa7bgf3valUUkd1NscnZUdXvQOkTRIvxP1u2uiZHjcEWIkwSQvlDzWcU3a4rWwjZZtqdBQA5mk8t8DtpJegz/+gOP6pulPxpWNqPz+V9JU6ObmigH6S/kmnc08ZJwGrmCeWOQBfop8JrJ9iAppiUmlmoyS9HmT2z+ErkuJa5lZCmGb5SDu7goaxUVtVvhgFLbOFOim0fAZPCnOimb0SDs+Jf1jLPhPH4ArjCaGNpYFL6fRBWBL4ppl9QH0+Gms26DAsbKTUkfQz4Gdmtp+kE6oqyzMsnmFmn1Z7wqmN8MCGCjScAdyXSG+dmbjh9zDpPUjpq4Ckj3dU8FhkP42KaqPUJo6Z1vctpZ02mNnH8VXAE/I4St83s0/jq4r9FZmah36bfF0A9gO+h1s2gjPRfUp15sKf7R10xsH6hJltJOnaiue1rJkhj5VVpbcpzGFHJrbRiqmGUZjZmTinvAz4oaR7K6p9vKKsQGsgvwi/Db+1Dl1dMJO3JcnMtsJnjaeY2R7hWIpCcZfQXpMS7/e4WKtQlk4ik8QwzRWMqKMNuY1/Ew0/S6VFbou+feh/nnB9df4VbxRMIuAx+vJYFHgVt6s/PjyPwmu4SXn8Vhd1CDSeYO3pJi8xsx0r6hwS1ZkALIZ/oMBDnUxIpKVqrFA6lvIeJF93BZ4G3hftN3kfp4yZ1vctsZ02HIbrwjCzLfFxsgO+av4lsGmqrwv0TRKDcl+KlMkRvtdAz/p41IOq5yVcd3mJmf0a9235Z6B9VuA4/FuY0kYrpiaHu//Rx5EbH+BkpuMceciONSSNsZbwvuaRN6/ARQz/h+sh7lYpemRFP0dL+rqF9Jtmdo2kjZvOqWlnYUnPRPutjkSTC2Vaauq8R51J5TfBfTzOxZ/7trioIU75OVZ9zm2fw5WYc0lapMf0/5aWdJNWk5NcbkFWTCrmwGXKHSFHJG3QS3p7hdIqZgRuPv24osjF1oNozTV9nyNp+wGc9wn8fQP3Y7nYIudIMzs10Hlk2O8qOnU45/24dKBYnf0d9/GpmsR2S/+nJf3R3HnxcFzUXkwsFsN1td9Wf4/yfm0k9Te1MIopBVaKW19TZ9LgDmZ6O+Jy3JvMbDHc6unMljaelOdCHouHifgSPovogDqTEiXTawkhwnuNxHt3qaQtrLuk8ntJ+lW0vwawjzpjDQ0aZvYArmupfamsISd53aSigBJzBww1rDMH9ds4k7ilQcQFkCz2aOm7dcxUnHM4roc7KxTtgIei2RKfIP0LnyB9WtLocM79klaoaK6WFvNc99+RdF3Y3wD4saQPRXXilckMuB7hn20T24q+ZsZD6YNHiG0N3d/NvZtqRE/vMqxT/JH0HK7kKvafpDNoWB0KkcBn8ACA09G8/G5rx3dcz/Npgmik0FWURCOTC1Vijg5I2iL87tZWNzrnV6X9MUBPmURASrrJW83s/eof3HCKZQRtCPqXKjSJuJLFHpMBWwCrSvofTNIBjcX1KuNwR70HIiaxGmkpRMvjd9aCSQBIut5K/k2xyNj8ZduKIP7qpi9J/8ad97pB6/tWIDOKAcDqM1gZnQ48TW2kKKFTFKkPAUeax9FpUqzVoTz7vZA+0UjZ8W9yo3V5a2azxbLesLKo0gFNDkbQhpR0k605yQc6yxwumMcnOxy3Youjwy5Ve1J37Q/6favAnPSlP50DQNKp5hFY58ct1go8h4uG21Aeh4+Z2ffo07HsTP/gjX0n+0r0z+YOuU0GMFV9DQTJbUyTjMLcI/FruFlabSC+BhzTcOzB0Efj4FaaJ2mrQtHMdpb0O2AFM3tfuZKkY1MsYSIsoijV6EBgDak8u6SlCvfjMtgC8TObCffm/htdwtoDKqakXP1BQletOcnbZpkptFhLetJQJyUg3W8lfTamr6LsNDzW2HF4DLDdcC/nA4DXFIVQD+fvgZue/jQqa0r/2vq+JbZT4HBgrHkmOsN1FQcBSHrGzP4G7GRmS4WV9PT4SvFJ6866anc8DP8F+Ji/idJKtiSeGwGsCfwnHBtP/buyQNSG4e/tU/0qJrbRhmlSR2GeFH0MnkRkpcA4bpW0ag/7uK7puKQNQ73jgT9I+usA+9lL0q+sOR/2rlXHojqTRAdmdjJwQpVopAuaJimPo7K7JK2eQkvLy/gdSbWhOczNAm+O5cAJ9B6Mv6DLSVrWPLroeZLWja+JlpSrLX00hhNRSz6EkkK+lRZrSU8a9seVx3z52ZWVuOY5K8bH8nrrM6gYr2CEYR7KBGBtlRTXZjZDoDdeRdWOmab70nSfmtoxswVxgwFwQ4HnomMn4SFINpL0vsBsrpJH/a18zwoomF+H+/SX4j1voDfWtb2NR2b4taQXzFOXNvVVKK+J732p/eQ2mjBNrihoTyY0aLQNkAhjgO+Z2XK4z8fZhWw0sZ9fhd8fNtSpkyFPgpndi78c0wG7mSdSqhSNJKA2eUwKLXg4hZ9QSgQTUGk3HmEZXHTQDT6Jm0DeBSDpb+YmjTFq421ViA8nnUOfxV1bHKdJYpqmWWYbLRFaE+vQkLjIzL4FFPkLXo+u5y3cKi7Gm4FZPWJm++JxuWYD/lNmEoHetyret6RkVwmobcfcUCTGuPA7g5ktJtcPgkdmXj0wZORxnGYI/2vfsxiS3jGz/5nZHPI8GHX1akVaqR/xgLvM/YTuHEQbtZhWGUVKIL4hgfocsebGlchHhkG7TMr5YUXS1H5qXoaFcbPGXiAlCU0T7sJjWY0pHzCzz5f2yx/p5/DwCt0gJaBibQKfFvEhoc6SXdATK4GLWeZWKbRESEmsU5u4SNLhwOFmdrikb7XQuz8eHv/LeEiWjXBP9F+b2QKSno8rmyfwKWOwYyalnUvpz6yF+3XMj4fUgLSMkin4BzDezK6m05kuNpdeBI/QW6xeb8Kd+57usq8P4uKyJ0JfA5ng1WJaFT01JhMaJprWwp3MtsItLposRuLzksVKLe10vcxvaa81CU3DucsBL6k6KU2/D89gYWZfx1cim+Dy692B3yvysrbEZEE9oGU+3C9kgqLouqU6KcmEqvxhdiqJK0aQkMDLPOrs4nQ6CN6YcC274Mzja4TVGh7v6ifAz8tjczBjZiDtmNkS+KTiI3jo9BNC+U74u7g6zmS2wVMAd5NFs/bdLIl6r8adUGOF906SNumyr0oRU69WFNMkowCwhExZCW18khBGOOzPiftA/LmLNo7CRR+P4iEy/lz+QFiaIpXomNSl/4OZPU1kpluGWvwxJgfMbHVJd9UcWwb31F0aj+T5DbU469W0Y3gk0eWJsn9JunrAhA8QYbX0Y3wsLAnsKakxfH5NOyOBI+UOmYPyhzGzI3AT7PvpdCL8hKWFet8cVxSvFOreBxyhAVjohWe1E+69f0gQJb1H0h0tp5bbWQZPRftBXFF+RiEiC8xzbdwiamOozihpZiPLDLWmrxnwsSXcge+t0vEqPVFV2cy48c1DDX2tQl8q1Jsk3V063tpGbdvTMKMY0Cyp1EaKMrCRmZjZXsAfmxiVpSkv18StT2bHB/erwO6x+MbcXnx/dVqEHCNpdzN7FhdlVOpqUmSz1kV4gyZaojrX4dYm5+Phxe+Njt2E+5vciOf1XkdSo4NXA92VisBwrGcpVxPouBfYUNKLYUVwlqR1ouPJtJjZbSqlGo2OnStpO6uxiFGnkvkhPGR4P9Gs9cBBsMsx06RkTjE3XwlnECsCR+FGJP0+9uV3uIbux4A/4hnr7q+p8zE89P+jgY4lgb1iJmlm1+Dv7R9C0Q54vvuNozofxydFM0ha0sxWBQ5RZHZtno72C/T5pnwSD3x5QmobTZgmdRRmdiS+tLyPPtmj8I9ON+hF4qJfAztaMMWrmSWlKC9PBfaWdBOAubnnaXj47QJNiYue1SCd6tSjxEVR2YbmnuvbAb8Kq6Vz5Caes0v6daj6EzOrXHkkolIRGNAUb6sDQfZeJAd6piwiCzPiteI6uMVN8WzfkvQigKTHzJ0fB0QLbv55Ee61H8vHL6C7BF6P4eahVTq8+4H5yh9K81zdLya03e2YaVIyp7RzNx5p+FL8OaxlkU49YrTXmAcDvCB+70pYBV9p/SZM3E7FDVFej+ocC60BK3fHdRRFVIVb6O+z8YNA7/WBznFmVtZ57YHfnyLe05F4TpoTumijFtMko8A9mZermiV1idFmdizwi7C/D31J2Qu0MZNfEGZJwCF4QLs/0me6B2nKy3cKJgEg6WYzK1u5jDCzuSS9AmCuQC9oGbTVl3VnAtpES3zOc3gwv+vw3OLfx/NXzBQYS0H3zPF+nciqBrWKQEkXh/ZqdT1hdvZL3HGrEH8tYmav4sz7LjP7KB4+/JG4DvBeM9tb0lXhnNg4oWO/+JA10RJhJjyfwkZRmfCPX6HU3ltSh+I/fGDisn8B48LMN3Yi/DL+ETqR/pgH1wHu2EZkl2OmVsmc2E6qE+ZeeCDDt83sP1SsbuSivF/jCvv1cT3DcWZ2PnBoYA6tASvlOoS2Wf1/Jb1mnYZiZQZmRLHDwv/4hJQ2ajGtMoqmWVI3SAkj3MZMamdJEb6IKy+/S5/ycs9SnRvMrVf+EOpsD1xvwfEvfDiPAf5qZufhg2gbPGIm9Cn/BoNkE9AWWgAwdyDcHrcGewm/z18Lh5+lU6cSh0IRnR/INmxad8AS5PDA6bhIoUNnZK5bOg2fff4M+IhKwRXDrO4yPNrqN0rNjynVTaGl+J/iSbwJ/S3ENi+VXRS2KrxXFeJaecyyk6pOqEA3Y6ZIG7CAmR1GUDKntpPIYJNWN4FhbYHP/pfAx/NZuI7gMjwZ0mgzu4zOgJV3mps/fwQ4TtIjYaV5Cj7On8ANa+KJzn3mEYdHmutXvgzcWiLpNOB2MyukF1uHNrtpo/5661dWUx8i2e7C+MtbNUvqdZ+z4szkI6HoajwJe7FEvB23TLkzMIz5cLlro4y0op8mBz9J2ijUW4G+j+i1dfLVoUAbLWb2V1zBf5768nVPTnrmpzMExZMpcngze0Q15sxmNkHSe83sEdwA4e3S8RmA+yW9t+r8Ut1knYB5nuc9cHl8fE27m9mX8FXp0nj48gKz46lQd47KChqXDbuTIr+a2UOSlquhddKxIJL7MbCQpM3Dc19HJY/tFJjZ8jQomXsBq46q8BrwRPH8go7iOjyH+62l84+X54o5raKdAlvjWSr/Gz7gX8ONKVbDxdWFUhpzh+DvhOPgebZ/JCn2rSnoXi/s3qTOXDFxGxbaOLTcRh2mNUbRK1PSnyoxcVFCW1WmeN+TdK71SJFqncmC+kEtXsEDQZg1rYfTfZP6lPdDTksbzENOHwMshId5Xxw3UV6xVK/SgiWIh5bGletFGIVF8VwhEyXta+7Ath3O+OI6nwHOlfstdENzmzXNeXh4ix1xkeZO4Zr2N7M58IQ5h9MZU+iN8v03sw3wcfk4/oFZFA+VfaOZXYrnULmsdM7mwJclbR72L8dnvN+RtIq5fm2sSgYEdWOmVKcx1EpqO00ws9vw97GITvB+PODjHMCXJF1lZutJurl03rqSbknsY5IhjJn9Hs/c+LOwX/aGr7X+GwjCamjWkj6l+ZxpiVEUCLP8/yhYPIQbN6MSQvOG+q25JrphJuVZEp5P4Z9m9nF5nPwUe+w58Q/TEnRacn3ZzC6RtKV5ILoqi5CeBG+LaDkRD3lcWHJsj4c+3meoaUmBmd2Nr2z+Imk1M9sQz8GxR1RnC1wPUWnBEj6OW9GpqL4o/ogGUVpVna5WdW20hDpjw7XcI2ll87wFNymyhDJXrj4t6c3AEFYGzlRkZGAeimNHBZNK8xhSf5CH7FgGV87eSp+YbE08OvKWkh4O59wpt0waq75wIh0Wg01jJqpzMO2hVprG3r6Sfp5wfy/AJ2v3hf0VcGZ7IK7jWbX8MQ/1ijA1KZkp78JFV6/g4qaNov4ekPS+qN1a679UBGb0RVx3cScwCviZpJ8kNaAe5KR9t214TubZov3Z8FhPveyjSH6+ftUWji2MD/wZwv78+BL9bwPo71ZcRr8b7hW7Kz7zG477+yBhEhL2R+Cz2WF/9jX0jg6/dxPydOPJo8rX9N5of2ngwclEz7pNZSm04NZU4JZ8K+GRbR8r1RmHTyreCzyMO8JdVqpzTwUt90T/Zwxj7piw7Q7MVKp/Pa7gvivsr40nC+pqzAR6jc7c2/ektlP0n3D/760rC/fpa/iq8IBo+0ExZoCPh99dq7ZwbEt8ovAcHtup6Gd94NKK/t+D6xVuwVc63+1yTI0LvzuF5zR91bOt26ZVZfZMikJVS/pHkOElwZojMkpuLTMmtF1pS25mX8FlhhOAGcNM6EhcfLFGqJOsvAzXVBlMr0bmGrfTs2VtQF0qzwHRYmazhWNVqSQL0VFHtrIu6X019HEjcJaZvUBkUhqQknK1irY9JZVjI5Xr/EDSD6KiE3DRBzVlKbScbO5n8D1cGT0b/dNu/k/S20FUc4I8levYUp3RZvYb4Hdhfyci81y55WCTLB6aU6EWqB0zEVJCraS004b7zJXxRY747YEHzM2V/4ffy3L+l9eLa1KCpZykS8y9qWdXsPwLGB36K9evs/5LxfRhVbk17hX/3+I+pmBaZRT/jOV+5hnP/t3F+a32523MBL/3y0l62dx34mF81hhbujTlCC7jt2b2BTzsdqygf5nmMM3dWgjVImJss+MvVkcqz1AtmRbrTCVpZvYipVSS1j9b2ZfNbB1J3+6C9K3w5/9V/EM4By5qKOTdUGPBktB2itnxmNDXOrhhw3zWGUF3FG6tkkyLpN+EvzfQaTkU47/mgTF3oS++VDm/w5dwS71CH3YT1SaxtZCbB69PRSrUxDFT4Fxzy745w1jfHTdRTW1nZesLcBijbP76OVzZ/5Wwfwu+ivgvsGaYWJ6ulvAY5uE5tlWnU+nZkjYN9+VtXPQU36vyBKXR+s/SAlKCO/49jq+abwxMKusommBmH8BnC3/Db+h7gO1VEYQuoa334B8q4ZZLz4XyxVtO/ZM6FVaT8vXW9NOmvNwHNy99lb6BIw2hzL9OZ1OgbnXV0F5KKsl76MxWNhIXTSQHQzPPj3CjpEcqjjXOltVFpr0EOv4P2BCXJf8yOvQGcDEezTWJFjN7FBex3oTrJu6r6G+F0NdfJf3B3FR3O0lHxvJ8M1ux6vwurmtb4ApJb5jZd/GV0Y8iBtJ0TTeENhpDraS0Ywke16Gt/RUUy+Uy6073OE4tkRtSYG79dw5u9NAz6z8zm04lK7zautMiowAIy7DCtG9Ayd7NY/N8H7gWH7jr427xp5bq9WMmQbxxdlTtM/G+OsMxpCgvHwPWUnMokJnwmdIkixDgl0o0keslUmipYp7lssAoNggrJ8ytqa7vklH8ELd/XxJf+t+If1zHDfDyMLNrFUySG+o8LGnZaP8aSRtbCLEx0L5DWzPis+kP40Evl8Nl0p9MPH+SsrZKcVtRv9KENhwrFOrr4dFljwa+L+mDXV5TbaiVxPNTGUWVonqs3Dig1ZAlOmcM8EmF8OVh8vintns5GFiFiXd0bAv6m0snRWOYVkVP4C9OkbpxdTND0pldtvENYDVJLwGYBxq8FXfnJ5SVmckJZnYILc5VJRwDraEAJuBetE04E5+dFm79O+JRK7dtOa8rWFoqzxRaalNJmtlVkj5KQ7ayVEg6OLQ5Mx4v5xt4/uQi7HSxsqiaQe4emFUMA5YtysNHMr4nhThqlqI83JsFzexDwPut0+u86KsQldbSEu2+g4tK3sHl6i+EDesi1lOJ3kpYhQmtme2qPme8wmN4C1xxe6mZ/ajURsqYaQq1ktJOY/TXIIbbEVjSPPxJgVGEtKlq0T2W8B3gZvMkUoYz7bKjLFYTd67hOU3ShUZtVJp444wBM/slHgp+Q+A3uD4lOZjiNMkozM3sNsAZxWW4N+rN+MerG7xEpxLxjVAWo5KZqMZRqQYpyst/4qEWrqPeiXAlRZnJgOvMrOcOd0pLGJ9Cy+7Up5KcL/T1BzO7nr6QJ99UlK0sBUEcsi6upBwLfD30FaMp5erjuLz3R7iuw8L5caj40/BUr99QiAFlZhPVmafi+7jCeRH6R/KN9Tcp6V9fx61jjsU/zvG43D/8Nuna5jQPaDkCGGWdyZSQx4wqcAzwUZVMaAlGGcAz5rqFTfB8KzNSCm2TOGY+COxsZo9Tk3OhpZ0FrTl/yzG4x/+8dOrS3sAjFE+Cma2LWzoVH/h+5t2SrjA33ij6/0p5xW99cec6ovPiq9qU51Tg0NBPh4l3dPxDYcJyjzzr5TF0TjQbMU2KngKHXgWXZa9i7jn6OyXGgLc+ReOquDPOhfjD3Qpf3n8uqnsrLhopnLNmwEUjrak6o5dzE3xAxsrLJyXtHdXdtaoNdfpa/A63eLgt7H8Q2EfSLu1XPTiUl/1ttFhLKskgavt6XX+lD1kbbXfhCYIuxZW/f1VLHDArpVwNH9WvAkdLusjMHlNJP2RuNPETPCDkz/GcE/10SGb2PUmHdkF/By2hbCtcrLcWnpXuVlwPc01U50hVxHqS9E1r1s1InVF+7ymvQuIyc4vCzfAUqo+YpyF9vzy+VdN1FeKexeRe8pV6P7UrlYt2Kt+RqJ0zQv1ZgX/Lg28ui+tFLi+J0x7En/cYohhLMUMOjKoxLLo1ROeN6qTQM1rSmuY+QauFupPEtGZ2u6QPmjsTfgqf0N6nhIgAMI2uKOi76W+bRyR9ATejS0Uxa3k0bAUuLP5EzGQCHoOlg5kk9hPPSJ/HdSDgkTlnjiuqOWhdsXSdHrjVzJ4M+4tTSk7fC1hawvhGWtSeSnIOfKZVF9snmVHInaRG4auKTXDT0hcUIpLWoCPlqqQ/mdlVwKHmyvFyvC6CbPsjwL44Q5qpXCfUO9Q6TX6vl3RJVd0qWkIbFwIXmjtzbo5b8BxI57ipjfWk7pT0Y6zBhBafoY8GsL50pB3jrmnM4Ix1dUlPmNkfJX26jpCmdprekRJuBD5sIYw5blG2fbiuAq+pPafGiSQE/KQ97lwKPa9as4n3JeZOuT/Bk0iJYDGWgmmVUYwON+3X+IzgH3hI3iQoLW9uCjMZiYc6OK6mn9aXNVHenLJ07SViBldO5dkNLU2pJJ8oyeQHDPM8BR/GGfGauDPVTaU6ZTPEfilX5aaNB5gnkFmHCsits443D7FRqVi1/ia/+5vZhxRMflNoMbM/4qvmR/GPxy7A7eHYpFhP1qlfmR03Be0WX6TZhDZOQToTbjTwEEF+HtA0ZuLJQJsVX2071ql36Af1WSyZpH8Fhn+i3Mt6XKn6dWb2E3xCEot6Yz+glICfTdF5C6TQsxXOEPuZeIf2ihXqH83sEtzvqmoCVolpUvQUwzwd4ihJqbP8+NzrqP44J/slmNkdktZqqdOkSF1Q0rMpy3Lrn1y+qPNkVflAYAmpPFNpaRKnlUVZg0F4cW7E9VR3agAWcL2E9cbkdz1chPZOVDajPFzHHCTGekroZyQuwli+i3NWx0Ocfz7sN44ZS7TASmjnRXwS8AecaZaNBQpT3LE4Iz0O2EPSfVayuLLqIJyK331LCPjZNMajOq30tMH6WxneDJykVItHTYYQBO+GDZfTHYsrrT45wDbWiLZ1Q3tHlepch1s8dWzR8eNwefWHcfvy1fFldtzGp6NtJzzmy/E1NM2DKzfXqDg2Hhd7jcfzIryNv+S9uqefx8V4f8VnuZ9oqJtECy4qWa6ifKUe0v2VirL9w+8y+CrwXjznwMJDMDbvAeaO9ucOZcm0UBGuIi7DLbp6EoIk0LRYl+eMTx0zuA7gdVx083b4X+y/3kU7I3FdyRm40cKPgBUr6q2Pe5J/M+wvVfe+tVzjTqGdp3Efp4dwP5WkMd4NPfj37BE8ym3HvQnHz8XDjm8Ytl/jcbKSrmWaXFFYQgCyQbTdsUIICswCM+Ef+7clHRiOt85MKvqYpLwMs+GDJN0blIR34fLgpfFUiD9taKdjZjdYWEsqz5Zz+9Fig0zf2AXdTXbzN9GjlKtd0LMDcAQ+yYhNfvdto8XcZ2dhXF+wI32z5lG4n8ryUd0Lgf3UsqI0N9ldgk7zzTOj4zfiYrQ76BQRfiIcj73MR+CToXkkbTqYMVOisat2zC2vdsBl9j9URbBAawgdY2bfr2pXJb8Eqwn4GR3vaoyHd382lSK/mtkEPMZUZdh1M7tfnVaGlWV1mFZ1FBvhuQEEYJ6/uWvPU+sMlT0CX1nMEddRf2/vW8zDCxTHK616WhArL5dUX0iL3YCrJe1iZoW8+ad1jcg9Y7tyempBWyrPWtTQ8gP6p2/smae59dnNL1WSX89OsJuny5SrQRSzAJ0f1Vic1moJoxqTXzP7VgItm+IhKBbBV8sFo3id/p7dc+FxjSo/8IHe3+KTjnF0mm/GpuTlGFJlxDGRCuuyP4b9AY+ZEpLaCeVb4ExiCfqSIcV1qkLH7KJO7/RYUTwTrnt7IGpjYWBB3AryQXNHuK/gz2ah6Nwf0DLGrSLyq5mVI78+X8ckAu4ys7XVaWWYklIXmHYZRS8Ch0Gnk9zbwEQ8WcwktDGTIC8+mCioHT6jeC2q06S8jOXpGxMsGeThEv5XoqVqZtfLhEBJqTy7oKUqfWP5mvZXTbiFBHpvo91uPjnlqpnthz/L5+nMxR7rFlotYcxt9MfJzWx3Bg40s58l0rKCPNf4dpLObbn+tg88uHJ/hWJSVQV5eIzF8TwRfzE3hx0ZHW8y/kgeMy1obcfMzsQj6V6GryLqwnX/CjhAnaFjfo3rG4r2OmKWmdnReDIgLCHgZ4TWMY7f/9fNc9dcjq8ux+CroQKjzewc3EIstqD6IT2weJxWGUU5cNha+I2+CDpnVE1Qp7NUHdqYyam4zLkI2fBZ3DlrkkhBzakZnwofqKfxj+0VAOZexuUAb00zu16gG2/zFFpS0jfuiqcZjfG5irIqnC/Pq/Av1XvadpNydX9c1lx2uoyRYglzErCKufXUAbhs+cxEWj5mZgfhH5NGRlG+ZnMF+A74ZKXAvXgstGepgXmAvj3xGfjSuOjrl4T0uuZK3APpHz5iI7obM01IaWdnfCWwPx48ctIl0OcdD57UZ5JIWNL1Vh2pNsYs+CoO/F60BfwskDLGUyK/jsItqD4alYkeWTxOq4yiUr6YirBsOxl/KcYDu9ct+xKYydLqtAv/oQXTtzBwjg793IN79T5TOn8PfGb6ETyw4auhvMjXXNC8Ki5eu69liTpgKD1DYCot++Ezszdx5e2VhNDKVh9uIRYbtWGEmX0bD7dxQPmgpGO7FA0+hSsTm/DfIJ4qxJ7z0X8G+bYkmTvN/ULSKWa2R1mPUoMr8Iiks5lHSo2nqvHHkND/avh93BafxJSZ9bzA/WFSFZtvxpOpffDJ1u3h2CNB1FLgLDyo3Za4CGVX3BcoecwEWpcEnlWw1AmToQUkPZ7SjqQRiV3Vho6JaInN0UfikQIK/cR/FKzH5I6CD9UwCegc438gpCgt1WmN/KpEvxcrxYJKxTSpzC6jmEkpUZltZqOBb9GnVPy8QujgqE4SMzGPDPkNhbSKQexwtKR1rEeKVHPF2874LOuDwOGRrHtIkUKLmc2immyDZrakpInhZVmSChNPXC7cGhXTzJbDZ2lfoTNaK5DsLxO3dwoeQ+xSOj+qx0Z1qlLfflfSeVGdG/AP/u64NdwLeFKcbswhL5S0Vc2xZfGVww7A3/GP+Ncl9TOxtrTgd4XX71i5AcB0uIVV4Zk9JqzcYm/tOyV9oKrthmsajYeiiKMc3NJtOwn9zIWLbOKAlT9UlDfCOs3R38Z1BEU+7eSAn1F7o/yQWnOchPrTyXOJJKVLtsR0v3WYVlcUKTOpJoxQCG8MnGeeD7mMX+AhJoqP/E9xRWMZXwTODLoK8NlgYVvdlSK1Advjdvn/Mo81dQVdeGX2GCm0vGZmP8ZfzvJs+48EL11cx9S1hUwBeWyiI8MHLDnuTQOeDNsMVHhmB5yPM8nCEmZrXKcRY3t8bO4elNiL0SmPboWkrcxD0xQf0dsLZS8um74JT1daBJr8ak07jfqHgBvCymxmM9sEt9e/ODpe6NGeNY9g+jdcTNUtplMUXl/SWxViuwHD3Nfgi7hF5Hjga6rxqZF7ic+F6zenAxYwDyx6F12I08xTHpxKEMWa2Wv4cx9jZjtL+l3VajfgWPoU6G2K6bZYUI2YphhFzUzKuhQvgAdM+1TdvjzOUAozQdLduDx6VNiPl5TJitQWvFnM0CW9ZG5iN1xIoeUxfCV2i5ntKGlidKxD6xfu+5G4FZjRX96cglvN7FgaDApSUKxArDkj3wXA1pIeDHUXBK4mUnIG5nAW8AEz2xJPa3pmRVu1MM8BcTRuTWN41OJvSDof1399BvcuvgKf7VaFQmnVPwQchItAxwN74cri30THfxQmQl/DowWPwj2Iu8WLZvYJSRcF2rbC3+Ne4Qycqd2EhzJ5H33JizpgZofiurBH6ZvJC899nSxOw/VPe0u6KbS7Hi4yXhko9CK1OkolZNML+G/xvpnZCEnXmdlPU4mcpkRP5lZAN+HejcVMql/wtoR2Tms4LLnHdDlo3dHxvhKC1lm1j0XcT5IHuJm9iq9sgEnhjov9ZOV9KsLMbA/6Ky93T6HF+pLU74yLlr5TfCit5PNgLfbjifT+EVfaFi/bZ4FVYjGfWVKAt5VwuXYxW/47JbPK8OH9GC5yWhR3pPq6ogB5ZrYdvoK4PrpHxUc+lZa7gU0kFaHF58Nnk6tEdWbFQz/sgCvCz8TzJcS0jCPoHxQ8ii14BZvZcmFVVnVP15WUHA4k0PcF+vtr7B7VWRrXdywU7stT+P2d0E07DTRM8nYO4rM7VO8F/hAe2PCtquOpsIoIA+UxntjOxfQXPb2GrzR+hUcc3hp/n+bFxU8fUEJwUpj2GMXW+ExqXVzkcTbwG6VZL3XbVysz6XF/y+LWMgtIWsnMVsa9U39UJ2eOiKmz+BkoLefh4o0dcQXfTrg8dP8UWqwzZMMS+Mf3GXy2el2JUdwiad1B0jtO/TORdZSZ51D+Hz5jfF8QO1wVy8ctISNfKN8H9xBeAk9AdWvpeONHPpGWcsiJETToOUIb2+IGERtH5bX6hzDx+i0e9fcfpfbuwv14msxqY3PpW/FJXDkaaz+RcNOKrZt2Ks4tT0KawoX8EfhS8YwGijCrnxlXZAsXO/6HEGBR7l+0JK70XoJO5hf7u/wMV6jHTsSvhzZH4SK1f+Om6EUsqN8pMWTLNMUoCqTMpN5tMFeAfgP4VTTzu1fSSsNAS/FRKTKbTY9njCvnF2g8P9ofgdv87wrMLGnB6NjPcPPNP9OpQO4mzHitQUFUp1jljI3ubznbXm1GvpKc2fAgfffgoSTKCu/Gj3wiLT/BxRfxh+MelcKKJ9ybo/D0urvgH6u9gfslfcfc8ucinMHsouDMFc4bS4OzZ7jmOJ5RP2YdHWuU1ZfuXW07bTCzd+hzpDP8A/4vKsSZZrYmfeFUKq3BqlZV5bIUqUGYOJyCi/b+Fx2MDQr6GQcUZWZ2H3BJ+dlbRZj5OkxTOooCchf63wO/j2ZS38RD+A4pzOxmXCZ+E27BkWT1UIFZJN1hnY47SflwJwMKBeCrQRzzHKUw2C24NN6RK7R/aGZX4p6sMersx5MZBZ0GBYab136uVCfFrLXJrLIsZ76gphzginCt8Uf+sm5okfQNc/1NESr9ZEkdHsiJaNI//DcwjCvx0NZn4Lmw/xdoOwc3yHgxbjDQWx7nl5jZxyRdRn+0yuoT22mEpLKSvgln4Lqxjo93CSfg1m21ZUrTj/5H0vEtdWazkLcDIIgjZwvH3qIhpHxC/9PmimJKQlhWfjhsa+Ozk5skdaXsM7PL8VhA54XZ5ja4LmbzXtOcQMvnceuklXHF3Gx4juR+JqhTEqzaoKA4lmLWGptVgjP/Hygyq0ygwXDHrQ/E7cQf+RRaorrz4mPrSdXb8rfRNAOeLEd4PuzCPDUWEc6Jiz4Xw0UbF+Dy8SvKqzvzJE8flfSlqOwNnCG8Rd9Eo2MWn0hrT9pJ6KfWvNfM1sG9uL+CB/0sMAoPQFqIEFfCpQCFiep9+Ep2fKm9HfGwPVdRE9LczD6GGxk8ik92lsRXf+/FmcEcdKY7mB2fmCZZPmVGMQBYKS1kGd2IPUJ7C+IRIj8MbIi/1JuZB8pr6iceKEvhfhsfwk1sJwI7S3q8G1rebbAGxXnCuR/HxTFPhP3v40Ebn8Cjx04s1e8I8KYuFOhm9lNJX7FqpWNZZNEaQrqOFqsPErkUnhL1p6k0h/a2oP8HaC9Jl5dFhKH+rnik1JmBxyWVQ1YU9e5Tog1/dE6rrH6oYG4l9yYueuv4eJvZ/+Hv8Rfp9M95A7hY7pC4FW7gcjh9pq1r4v5ZX5cnnir6Ohw3sHiUKDSMSsYs5nGsiqCPD0n6j/UopHxmFAOApVk9JTETM3sUt475PT4DHae+PARdWz0F/cuIWIRV93GKGurJi1YnQ476ObbXtFiD4jzh3HuAteU+HVviduk74JFQt1XkRGlucfO0PJ/DBvhq6UxJr6YwATNbQ24bn+LAdgYequHOGrqbaJn0ATb3bVheUZBIdZHTIrTxIJ3+FksDl0pa3sz2lnRixTlL4SE71pf0vpp2Hygfs5asfimy+pR2eoGad7PQKVwjaWMLScVqzr8b2Ko8kTM33rhQnfqmCXi8p34WVma2kaRr6743Fau5Ds9sJeaimSZ1FIOF0tzlP95wLJahH4+LGIoP1A1mdqOkRxPllwCY2f64mOcN4NdhNXKQXEF/dGo7g0SKDHlAtFhNeGXgvZK2NbOt5AmNCoabAqnPA/xTwClBPDPGzPYu1f0jsKaZvRc3N7wIZ+4fo08nUXtthdin/FGrwQeBnczsCVy5WihTi498Ey3JQSIT8YYi81Nc5/JGaLMfkwjljwFfNLMbzGwtRWa7AOZOZmW9xRG4uC3O6reupNj/qFVWn9jOoNHybi5oHpr9/dbpB1WcexfuPPh4RbuPmxt/xLgXmBM3aS1jfTzHTdX3ZtJ3Jqyej6XkmU1nlsFa5BXFIBGW5mWxxyH1Z9S2MxseJvzrwCIqKdaCPHOFUj9nRscL65pN8SXvd4Hfqkt77CkJVhFeGegIr2wh/4d5ToS9ccX5HUrwjQkrig/hyvCJwKclFbmdO2L1W5+l0YF4zvUTyqIXa4hkazWpaunPBLCWbIVNtIRVzVV4kMhT8TD0r5rHRRo9AHHPSfhH5dxA/7a49/lfAk21YlYzWyucdzp93slr4hZUn5F0e1T3Hlqy+iXK6lvb6QXMdTK70F8M9mUL+kF8Alj2mC5WHXfj/j8dM/rw7C8uXff1+KrxTiosrMIkahs1RAsO/W1EyTNb0h5158TIK4pBwMx+iUeN3BC3BNkGT95SrlfLTMzsGHxAzYZHjfw+/fM1HwxsgDOKy3BrhZvpzAlQzFo+hosh7jPrNIEyDzJ4OP0ZTlcOh3WwzjDP/aBOu/kUWlLCK59srkT+Hj6zno208Nng5pvjcHvzByImsRr9o6X+1zwQ4S70zd7KM7+mSLbdRPFcEA+Y+EagZxTuJfxEAi1JQSK7wEx4iJFCZPYirn/4OC3WZXIrvLXwoIGfC8X34hF0q2bHc9IX0HGOiuPvx2X1G9EZxr0sgm1rpxe4DA9T38/qSe4Yeb6ZfU99uarLOBj4i3mompiJHkR/S6SDmwiR9L8waWiKFjwoz2w0mdI5TgsbrgiNf2fDLVTiOr/EP+hPhQc+HhdxFMe3wZ3kmvoZjzvK3B32F8ATFMV1TsNnWo/gzGt2YEypzs24OOIefJb4AzxURa/ux65NW7e04FYg0wPn4fJuonvwUzw0+6BSk+IhKVbD9TpF2YKUUnviDO14PHgkuFK3SE25Ax7b6BWcWRXbdbiiuan/9fAIsXHZWMJqP+yPoDONaS0tw/guzAfMl1Bv9ZryHXBGeDpuyTURZ3RxnQl4Jrim9lvb6dH19ks1W1PvE7hI8mhc1xMfWwX/NowJ25l4RIC4zkgSUtbiGRG/jnv7z11s0fG/4N+nn+Nm1z8Dbk2+3uEcXO/2DRdxgM8sFgJmxBO7x3UamUn4COwMfC/sLwasVdPPGFz8YuXBQ1/ynznD/jzAyqU6Y8Lv+HLZMNy7Vlrw2PzP4LM3wxnKTeHYvrhc/vGw/T6UdXz0e0zzDLgI4P3xByvQtQGer3n9aFsdl0WX21kNXxU9jjOT/UrHx1Wcc08KLT2+3s3x8Cp/D9sNwMei44Yz+L/jM/hX8BXH9xvarP3A4gz6E2F7T8XxPwPzJ9Dd2E6P7s1X8VAhC1LxYQ51DsfTn+4etqtxT/1u+7qQlpzkOEMsb49Fx2cN34jp8Inbl/F0tEk0ZNHT4HBxkFX+BDdDFP0jof4n/P7LzBYCXsIHV4Ff0Jfx7FAqMp7hSZXmDG2PAf6Bf5RiFDb3K5ckTjHeDPLMR8xsX/wjPFtd5W6RYv3TDS1yxWUsznoiyFaR5zj+eeh3IVzX8CH8BZ4PZ6g9g1WYiZrZXpIuV0IkW+suIOVjZvZl3C8BXPdSOO410jLIy4zp/QLuYHcgneabR5jZIpJOxu/1unjMoInhvKWAk8zsq5KOq2q61M/y8lShhS7t6fC7kJktpM7Al3MCD5pZP1l9l+30Am/h7/136AwKGItOt6BTX3IGvlosp6Rtw1y0pKxVexii6fGVKMCf1WXAy6zMHiDCR25thTg95jbMM5UfgLmn7gm4mOUXBGYi6fvheGM4hqBnWETSU2F/CWCUpHtK/cRhnWfCA7mNUWRCG6xNHsBfuEPxj+lPFIVeGAysOxPQVlrCPf00/RWGhX7H8Bn1h/AP1gr4jPav6jKXRMK11ZqJRnXeoO+jMQP+cv5T0ijrIiCluQnj8fjkQfis9Cvqi/3USksPrvd+YD2VbO3NQ8PfLI8xNRaPSfX3Up358NhTHT4W4djWkv4c7Z8saU9rMDeN6taOq27a6QXMg36uVb72Up17gA2Ke2ieFvl6dW+i3Pg+hfGyD52Oe7+Q9EJ4h36FBwScSN/K/E/AF5UY1DCvKAYIuQLpF7gYAUlv0pmrtmAm18gVin80d4YqM5PGcAySZGaX4R9EVONAJ6nDPM7MFiWKtRP62F7S1/EVSYqJb1dQn+fvYvisJfbl2DL6n0rLhXgEzDH0v7dX48xlHC76+7EGGEHWzH4r6bMtZbVmogUUpawNTGwrXIkMXYT2DgzhMw0kt9Jig4iiWjRRZhLh/JeiFev0VR9KSS9aMPE0s/eEsucCTSPMbEWFiLqS9gy/rabgasiN0U07PcIE3FquCYcDYwPzMty346DmU3wSpciHpum6zeOS/R7XyRTGLWsAdwQjkE3xCcui6jOOmB2ftH6PRMOPvKIYBMwTqv8VuEA1N9IqvFdLx1NCQzQ6YNW0a7jlTGzieZsSA/MNBuahxB/Hla2Fx3A5MmcrLdYQ1NDMfoXL6P+NM4q/4iuJrvMTVNA2EtedxPduQGai5edvDQEpLT1bWSstNogoqqGP24E95flS4vJV8BXxWuX7Vqp3Fz6TPQj/SB6JWz7di4tJj5J0SumcD9GfscUm4JNyY0ha2txy7peKot2mtNMLmNmf8Bn8dXSKwb5cqrcgfWLkOyQ9V9PeCvSJJl+VtGZ0rPa6zew2PIrt2FJ7q+L3f1Z85fOv0vHZgNvq3q9+9GVGMXBYX1yZd/APVlWUyRRm0hgaIoga3ovLwascsCh9XEYAq+IhFHaO6pyEW/mcR6ess6uQI20IIok9cEe0H0g6r+KD2UqLmZ0MnKBS7JtSX6PwWfuHwu98wL2Sdk2g81u4vLiIEgp+b9/Cg+h9K6p7WkNTUn9v/BG4TH99RVFoS/0XASm3Dy/9xyVdbB4Go6qTM7qgZZwGGEU19LEe7rR2Gp3mm7vi9vc3W2e01Y7TcfHng7jz4Mz42H1vWFnMhYeLn0Sfmf0WT4w0jj7GJnWaVI+jJjdGN+30Am3PKNRZFzdM+Kd5bpXVcT+gwh9mCfqYw39x5r9mWWrQdN1W8vcpnXc/nn+9UtRVvndNyKKnQSAWNTRgL+AA4B0zq2Qm8mxnDza0UZVCtYzYsedt4A/qnzhmJlyZHstrRXeRVlMgecyb9YE/mOcPL0fmTKFlPeBzZjYRn7X1Y5Ch/F84o34TD6iXlB5T0uHA4WZ2uFo8d9W9N/7b+Kpqq4Y2X8Hjc50c9pOylSXSMuAoqqGPm62/D8T9uF7uuVCnMdpqWHH8CzfkeDQ67xUzK0+a1sT9Zppmrm/K058W7U9H/5VXSjuDhjwKwAzAsqHoIfVPm3oSnr1yFfwbcAq+glzfPLT9KFwE+Wl5/KeJZSYR0HTdZmZzqRR40lwfMgJQYMxVYs5kT/3MKAaBIN7ZCfd+PTToBRZUFLKgjplYp+LTov/T4eaO8bL5iTDDW0bSaUHWW7YQavy4hDo910vU4NnQ39/NPcWPBDqWuIm01Ea+NbPj8FXEMrglyV9xS6Bd1edklopLzGzWqplfqjgo/B/U/TWzi1qq3JxKC7A/8G0zewtfIfWboLRB0vO4A2gdvbPgjlz/DfvL4Q6fj8uj3crMpg/Ht4jOmwn/iMW4F88rUnZ0jHGDNefmTm1n0DCPsXUGPhkwYFEz21XSjVG1tyXJPADgLySdYmaFJ/Tz+Ip6AXwV/AjVnvvg1/0dqq/7OOAqM/s6bnkJrqM4Mhz7Fr4irGIUycw0i54GAUvLNNbKTEK92fDZ2164zPpr0bGD8ZnScpKWNTcHPU9RZjdL8HS2hix4vbonFdeEqjORJdESZmMfDrs3KcjMzc1Hb8GX9u8wCJhbp6yC6zxOx73st5O0fqo4KLSzFO7ItDb+Ev4V+Ko89lEKHS/ijpl/AG6n/8s9KpWWoYB52JQ9wmz4vXhUgrPwMXgncCLwN0lvl85bGHifpL9EZdfh4tI7qE8ENAIXaX4UvzdX4hkq1U07vYCZjQF2VEgFG8bzHxRFyzVPJnYF7kPxYTzGUpyAag7cyGEHfMIzJ7Bpxbeh8brNDUUOxHUmwld+PylWqD2BJoMzyrSyEZyH8FgyRdndpTon4RYGD4T9uYA7o+Nz4k5LjwE/osIJBpe3WqmfsgNWiqfzDbisM27n3slwX1bCZ/lP4ErWMcCK3dKCz4rvxUNSHIJ7qO83GegtnuP38Q/fpLIu27kNDzExXdh2xuXK5XqLAx8J/2fGk/uAi+c2w2eqY8N4WLFbOkJbRqcj56KUHDl7cN9iZ8lDCR7muOhvfJdtrV+1DYCmnrST0M89bWX4yuYA4MNhfzE8E2BVe/PjIdRvAZ6Kyn6K57s+HJ8o9PQ6UrcsehocUrKefVDBTwImyWdnME8o8zXc4ulUYDXVO8G8JUmFXNfccqaMmSVdY2YmV5b9IMx6YtHBUGXBOxk4QJ25o3+Ni4q6oWUP/P79M7RzJD5LP6HH9L4RFNufBT4cZnCFeWc3YdFnkfTbaP93ZvaNuL5FFiy40nURXGS2sXxldAWe4W5GfKZ5vZn9UNLPu6TlRDodOf+BT1gqk+10AzM7Wm7aHNOyESEGl1ye3lWkWjVE1bX6gIrFuStH/2vb6TFGm9lvCLmtcabcEQBQrrg/C/hAmPXfoRrrK7lJ9AnACdYXFLII73ECHivsePpnXhwSZEYxOByPO67Mb2aHEUxbS3XqmMkTuHPYabgydo/4o6koDzBwrrk56JzhQ7M7/T3AU7yu/27unFXQsg2TR5Y7a8EkACRdX8HcUmgxItPO8L/W7XwQ2B7PZ7F7eLkXoy/wYGtY9KA4BLjczA7CFZSifwpTcPHiWrhoCbnYZv6orRlxef4OuIlnMcaSaIlQOUHp4vwmbIfHFbrH3KrvGdwq7yoA8ygCSSjp6gj//46bnX5T0kv0BVTcJ/wWzHhn+sZPSju9xJcCPYVe6Eb6POkJNG2Hj6Pr8XF7gpl9Q9L54T2+Pjx/wyeLn8K/C7uG3wUlfSc0d6W5yfGwIOsoBglrN22t9JOgT55YCZU8i4MSa5KMUtLVpeNlT+c5cFv12NN5KYYgC565jflddL7Qa0j6ZDe0mCdC2pW+D+XWwOmKsrSFj3o/KDEhS9TOAnTau79QOj4rHtI7Dl89ozzp0UT8WVYqDNWpJ7pd0getLyz4dLiYa2UzOxMX210GnC3p3hpaa2mJ+8Hv7Z2BYdR6S3cLM3tK0qLmocv3x0PSnKo+/dGHgKXj1VXoH5XyZ9e0Pxc+c/6QpG2j8rFl+q3Zl6OyncEgXMd8ku4vla8IvBBfn3lo703U51E/Hx7mexUzuxeXIvzXPHz61/D3ezXgYEkfDudvQN+4ui7eVxcZ6kL/I3HleWwok/aeDJfMa2rZcLnyQrj8cTEqgnfh6Qn3wYPWvW8KoHlWglx8MrU/Fz4TvitsPwPmGggtOIP9cthWqzg+HtfLjMctR97GHQ27oXc7fAZ3Br7cn4jH94/r3IYnTir2Z6OL6JvReUfhvhsP4gnv/wQcFo79D/ewfgMPfV5sbwCvd0MLbkBxET7bPwx4CM/al0rn3DXbPHh2vbrz5qVvAmp0GTSw1NZdpf1xwLrR/oeoCKDY1s4gx/bZwP9VlH8Y+H15bJb2RxRlMd24Z/X+0f7Y8Ps4rrucWLE9Vmp7Adz89vKwvwJB3xb29wvP4b7wroynQs9St+UVxSBgZvvhocOfp08sIpUcXAbFyf38T+HmbvOHPiaZOnYjt7aW2ElDiVRawqxw0VKd2iW4eVC4vSV9vgtaamd+UZ1xKjmwFWXWRTrKIGb4PA2WOwn01tJSKitWuwDXqrsc302rJCQtaWZr4+GtX8ZXsb/FGcUIPFfGCriJ854qBQ0ErlB10MCi/+nxWGWxU+kauIhmjkDXK7i4sGk89GtnMDCz0Yq8pkvHOiIJmNlPcEu6P4Si7fGP8zeDGGmLcA1P4JaT94Xz+qWJTaDrclyM/R35imU6nOEUFlYTcHHkgERwWUcxOOyPm6zW3vw6ZoIPoFQchWfDqnrRu5Fb18ZO6gWsxQ9AncrWVlrM7FBcdPAofcxQ9E9UE/dxl7mDXzcYoU5R00v0t/P/p5mtXnyUwkfr3+HY+qSloxyJr3aWp7+OqRs00RKjiAkk3LoqGWqPRgoevffb+If7WmBzSbcFBlV8HDuCBkp6zNxX5SrguBrmOhf+UT2/RNMY3IFtjrA/yfijm3YGiSYn20mJrMKE4HhcnFlEdj5Z7l8CbmQyGn8+F0VMYn2iSMFdYF5J55obZSDpbXPP+QJP4e/bgJAZxeCQcvNbmUkCnm+YDd5Ptcy0iKQaYxFJmw2CjjasQ7MfQLe0bIfLumsjXAY9RoEiJ8ff0sidhCvM7Eo6Z35lJfRXgPPM7G/4db2HELhP0sHBkOByNaSjlPSOmT1kZot1s6KsQC0tBczs+3h4kD+GOqeZ2XlK9JmxvnDdlVBf3udCgX2Igj5MHu4bEoIG0p+5CmfUP5N0aQVdk7JFhj6KVWhX7QwCE6zC493MNif6wEsezDPM6PtFPpB0ibl10+zq9Kq+Ex9/3eKf5pF9C+X+2nR+mx7DLegupdO/5FgSkBnF4JBy8xuZSVCi/hhYSNLm4QO/jjoDpo02s3PwxC1xPxfgpnNVSe7nwZXmO0Zlt5rZ+9UQO2mQeA8ud98h9Hsp7oR0X0XdFFqaksoXiGd4b4c+kwLfFZD0jTAjrZr5FXXuDDPl5UJRR8gGpaWjhITcAgn0NtISsBOeLe0/AGZ2BC7jT3WuPKaJBDrTkUL/FY1wj/A6vAXdebNbQ+rhbtoZJL4CXGpu0RTHwFqH/ulu77JSJNgCYVVl6jSnBhfHvoPrLbrBAbhOamkzuwX39t4mOv5k2GYgMcRNB71ZRzFwmHtM94MiiyUzOwV/oSuZSZtsMdQ5rbob7d6lzPR+3IxxIvWxk3oC6/MD+AnwQ3miofh4Ky1mtiYuorqXFi9ba/ACT6R3AdxsVURWTxZCeIT/26ozqu+PJX072j+CvqREMRN4OaqzflX/SrD/75KW64BPKoQzMTdZvUA9zMtgfUEBjf6BFYswHbVBAyWVc4639XeP3Dqs+J0NX8V9uPXkHiKM7R3pC0tzH67I/k+pXm0wT3OrtI3L49Xcou1GSWtYn9l1JdQ/V8h0+LfGqJ48YGazqBRJNgWZUfQATTe/jZmY2Z2SPmCdiYv6KSYb+n5I0nIpx6zPkadMyxMpfSXSU/YDuAg3nXymVK+VFjO7Dw+VPJ7OHB03RHVWwpWoxUv1dzzeU6VpaQ3NZXv3DwOFvfsk80vrH468vD+x+pL6JycaCLqk5c+4fPxqnPltgs++nw5ENUZTNTe3fkohkJ+Z7YLPdp/AIwJ3ZZrZC1ifafFtuM/BS7jO571DTUsKmsZ4+XmVzisYYTdm19viBgJvmNl3cRHsj9Snx1oHt4qaTdJi5qFx9pK0d9LFqEdmY9Pihi837weeDPurACfW1J2lpvx6XExUhJFYG7ihVGdZPMvZvWF/ZTxnBfhK5WMV7W5OMJUrla8H7Bb+z4fHoOrV/TgTN4f9EbBSQv1GWohCnTS0cSuwYbS/AV2arQJ3E+ViDrTcHf6PjcrHls4r789U0fZMpf21cTn0P3DxyztEpq8tdHZDy65NW0JfdxFyQOMJd/6GM4pDgfMT6Z0F11MU+8vh6VM/OcDx9T1cFPlp4DncQfPQXo3fXm/hWc8e7Y/CLY/AfZ5mrThnduDBAfR1T/hdD/e32IIofAyuM1yUAYbvGfab+W7eUm4+LcwE5/y34HqMW4CHgZVLbdTGRcKDiT2MB7PbL2xnhLJlS+0cjEedfDjsLwTc0sP7keQHkEoLcCwe42adcJ9WB1Yv1bm7go5+ZS10N9m73xWVl+36G/dr6ozGxRFjcYuX3YDDE+nshpaZcNHISlQwsIS+7o7+/wJfRRT74xLbuBGPeEy45pdxndo1wBGluoU459u4RdD3afC3CPXnqClPbmdybuEZW2lcFRPCrwOXA4tHx5fAJ37fqGhrLvwb8H/FVu4r/B6OByucVBb+315RlvyeZGX2ICHpKeuMV1SOZPpTPJ/ERaH+3Wb2f9H5Rd6GJtlibVwkeQiA99MpM70BX1b+p7MZPol7ft4Vzv2beVrEnkBS2aS0CSm0FF64cSa8QpFa4DHzvOSxF3i35oVNVk+rmNnrBDl8+A99cnjM030uHI6vRp+oYBQ+q+6ApAlmNlIe2+k08zAbjfkwuqBlOtw4YndcTGR4COxCD9ZPbl2DkWY2nTzy68Z4fKoCqd+NuSQ9Ev7vihs27GceSmQMnWlBU8yl9wHOkvSqpDfNbBYz21vSid20M4QwhS8yTDJ4mC78P9rM/gHcGHQthk+ojpBUDgXyedx6chHcIGFtPOZZ/B48Yx7mZxPgyCACjt/Hp8w95hUszvbHVzVJyIxicEi6+U3MJJIt3lfIFs1skmwxoDEukjxf92lRm1tWMAlICy44VGilRQ35j83ziO+NfxB/SJ8J4o2hLBlqsHpSS3KegE1xf49FcGuh4mG/js9sY/wrfCjHmdlR+HNMYrCJtPwEF18sqb4cyaNwf5uj8TGagj/geRD+jls03RTaei/p9viK/rcFDUwxl/6CpF9MatzjV32BTqu/yWoCbvUBCqsMQx4zD4dffPj3ptOE9pfAL4sJkqIc8yXsj+ubbpO0obnF249LdbbDIw8fLelV8xSscUDKL+IREhbGxYhX0hc7qx3DsSSbWjbcC/Us3JnuBTyS5DylOufjoQbuwh1yvo7H8SmON8oWw7Gl8HzI/8JDMtxMtGStoKsyZEHo+1f4YP0CPivpedjuxHvXSgsuj/4yLoI6vtjCsW1x8dp3iOTgPXqmNsBzD6woK+tdFsdn/6Nw8duxeIrQXtH/SBX9uJjrkS7bWhtf+c0alS1LSfzXcP7vcOb01fCOzBI913I4/pOB97e0N55OUU7hwNhVO4O8v4s3baW68+MhP14I1/97gi4M91yv3Urt3Bl+x+Exvai47sWqtl5dd7Z6mswwDyf+M+Aj+KzjKjyuy0vh+Fh5cLjDcbn4760i+FmoOyvuRVw38yjqVZ4fjjUGFxxKtNFiZrficY3KVk9nhOOz4QrOzXDRU1yn1ZHIWkJQSLqiy+vpZ8liZmMUJbOZ3DCzhyUt2+2xyURLN0EDU8ylf4J/kH8VivbCLbO+1k07UwLMsxRW4RPAwooyXJoH2dwN9+HYCA/7Mb2kj0V1ipVOIYZcEhdjrxiODy6pVmYU3cM8UuTSki4K+8fhYQwAfq6G2DMVbV2CrxI2wZW1/8bt+Fcxs0WAJSTdHOoeQF/o8N9LmlDT5loqZcl6N6LJhDAcnwGXc++I+y7EjOKHdedF54+mLwTFyZRCUNQx24p2lse9hY+ic7k/CldMrhjVXRcPlLc4nfGremVC+2fcX+LMUvnOeNa+ZMe+yYEwcXpJpQ9Porn0CFxX8pFQdDXwa4Uouqnt9AJhknEC8D7cgW0k8E95/LXk9LmhLcMdJL+JG74cJumemn7Xx8frFWqOWNAR8yyYFP+CPj3cZ/AVfFq4m8m1RJuaN9xa50PR/v24yd5ngT+HshXx1J5FnePwgGanEi3dcWXnp+izDlkQ+Gj4/wdgy6juQ3g44u/hSr2ifBTOuMp0rhx+9yCypMBt6QtrpC8O8b1LpgUXWXwh3JNJEUzDsc3CfT+CGtPjBFrGRf8fKB0b20U7W+E6opfCb7Edj3vZx3UfxE2X58fNouehIqvhIO7vwrg13vW4vuQY3LjhDnymOpTPeu1AxwW4YcK9uFnrC8BmNefMT6LoBPd3+cVg2xngtdVar+Fx2aDFNBmfKHw+jInT8VA/VX0tTZ/IaQNcHDtnAo1xBsKqjHzJVk9DNmimpg0YXdq/Lfp/c/htZSahvFa2SH+Tx7HR/5vC73a4cmoc7iH6gahOYYp3Z/wxos+UbiZKPhtDcO+SacGVba/i4ZYnEoVXxpWrA0oTWr4/Nfd6IKlQy0xhJkqhvalIjTqZ7vNG9JlLbzyUzziiYTQuWtwWF5esHcqXp7/fxydw/co/w3P+HxXh4nGGc1QYE9fRX6+V1E4vri383hOVje3i/H1wHdtJuNSgqe44nKm8N5zzE+CyUp0Dou3ruD7kyuj4kfjqewl8NXsgbko7afLVtGWrp4Ghw4xTUmy+WWQrW1DSrVH565L+CGBme0Xll1IhWyQEPiv1u3H0f97w+208KdCzZrYW8Fsz+5bcaqewvjF1BiU8L9D9nyBHHkp0Q8vXcEVvVWC5XoRtaDU37QaS/moeIXZT3DN9E9zw4DzrC7J3XZC1X0BnWJJkcWUiLdfiEV2HE21BA2Mciq9A/iLX2W2ImzpjZsvi93MH+kKkmKqt4mrb6TFqrdcsLYryCfjKaj1g3eh+VOlU/iePBvtJ4ARJJwST6hhtMc+2C7/xtwdcBCXcYKYWmVEMDH8zsw9Kuj0uDHLLInJpCjNBUUyn0MbquBkdeC7nZSU9HOq+HOosj4tqAEZKejYcvyO8GJeY2aL0yUfnLNHy49DOCPoYzlChG1om0Bc/qOdQmrlpEoLseEfgY7iYZ11gKfWFdikH2Yvjc4mG0OnvYrQFDYzxX0kvmdkIMxsh6Toz+2k49iC+gtxSQS9nZl+t6bOpnV7iszhj2BcXkS6KSwwgLYpyShj3Av81sx1w0VURJbcjTpZadHJKCxtfi8woBoZvAueY2ekEhzFgDfxBFiGCU5hJP6gzn8LB+Ef/sFI/36bPHv4NM1ta0qPh/GfNbAM80myhRL0q+GaU83kfQshzPITohpZ/4jO26+icfTfGKRpqmNnTeGTOk4Cvy+PtTIyYBDWz36kd3azYXg1WbDcBZ5nZC/QFFPwUPvO9zsyuwE1O60LYN7XTM6hPOf4f3I8nRkoU5ZOBK/AwOw+2dLcb7gdxmKSJZrYkfQ6mAJgn2zqQkiRC0kbmOdn3oe97cB+u22mKytyBbPU0QISbvy/9b/7z4fha+BL5dCqYiYJVklXnU5hH0qbh+Er0DQBwheBPFILemQf3+qdKFlDmDoDbSTormNX+BnfauTtUWQWXIX9eA4y4OhB0Q4uZ7VrVhoJ5bKizuaTLS318Ue7MNCQIM9at8Wfze9w7eLw6g7Z9HJdnPxH2v09fkL0vq8d5y99tCOPi3/g7sBNu2XNWLKYMdbbCP8Ab4bHF/lSIt1Lb6RG9SdZrVhNF2dybf7OwLYuvPK7ARWb/jM4fCZwpaacWeq7Cvzdfx5nKrng+movwMXk6fWHRi+/QTpJuSbrezCgmH9qYSahzcHTK27iS7o+q9qweLD1LRbTcX6xChgOptAQ5cGH73y+8ibmvxXeDTB7znBAbStp88lBejWDiuAH+UfgY/oHaA1c6/sPM7sGVuf8ysy1xR7sdcOXstsXEYFpGMG1dRtJfzGwWXKxa6TNkniJ3W3zStfFA2xkErQ/iIqcxRJEW1OcflRRFOdQdAXwQt4bbGGd0V6kvpPzNeKrUJnPYMfLQ5PcU+g0zuzPQ9iVJY0v1VwV+pUTz2MwoMqZYBBHaGTjzNFwOvKukG6M68wKX4P4Lm+EWNTs0vVSTG2E1Vyi0N5U0r5ndrZCD28xOxZnekWG/0V9kWoB5KI49cQucpc1sGeCXZSYwVO0k9HN73UfWzM7E465dhkdhSA55H86fFx83Z0XtvQ9nNnGek2Ojc26TtLZ5zLLjcfH2+cCbklao6ef+umP96mZGMbxoki0OG1FTCMxsDB4J86Gwvywu612jVG9+PMTJGGB3TUGD2sxmlvTvsKL4EK6cnwh8WtLoUCf5hZ1aYWbj8Oiot6svL8v4srHHULWT0M8RuP9EP+s18zhWxQc9HouFRdOoqJ2j8LD8/8ZFTyvjHtO/i+rEUodJUGeCtC1xvcyiuEXVKFx3ciRupv9Kif658XD8y6dcb1ZmDz/OwmWLW9IpW+wagekgaUDnT4GYvmASAJIeDrN1zOwN+syKhXvHLgVsY2YdL+NwQlJh7fNT3B7+ddy5r2ASqxEFeJyG8aY8WCAA5lFWB8Lwe9VOG4rVRD/rNXUXRfmjkg40N319HFfc34jHyfJG+5Kc1SZIk3RJ+PsaMMlwIuhCrjKzr9OpKz0SdwJOQmYUA4CZXUzn4BNu331dPBNIxDySTjGz/eWZ224IssW4v0XwWcJ6oa+b8HhRTwfZ+MG4LmSEV7e3cXvrQ6I2igBqSTOIyYkuaBltZr+h76XZGVd6I6ln4dGHApJODWKB+elT4oN7Ku82PFRNUbjBzL6NW0dtgpuIXzyM7TSih1ZsxTd4C+A8Sa9ZycfEoux0QEd2OqsJExLR+WUz+xvuX7JiqHs/nv0u+b5k0dMAYNV5j+fGP2SPSDoolZnUyRYlLR3VuRq3XIhzLuwkaZNgNbU5sKekiaH+Urip5hWSjovauRD3ZH1ykLdg0EihJSgE96Ev/PeNwEnysOpFnU8C10p6LezPCWwg6c+TifQkBAXlbJJeb62cUdyvPYiCRAK/6VaM2Kt2EvqZA5+gFbllbgAOKcZhF+0cgVvM/RsXmc0JXBLrP8zza28DXBSJ0+6VtJLVWAYWUGQhOBhkRtFDhJnyGEmrpjCTcE6lbFEh4GCoM06lHNpFmbmH5iYqeS8HMdRVigLbmdmNuJXNHXQqxYY8UFwTLYH2+STdXzpnReCFWLRWc2/GKjGgXy9hZr/HxYfv4KFKRgE/k/SToabl3Yhg4bY8Pql6aKAGCb1qp6WPP+Lm0MWH+LPAKpI+NYC25gZek/ROsNIapZCrPBwvcoWPjRjF3fLAoTPh6VZfLLU5H/CGemQ9mUVPPUR40MX/G6rqmLv3T8ruVSdbLOEl8+ifReTHHfAAdOBy/KoQFy8W8vwI30u8lKFAEy0n0JmMpsDceP6JHaOyKnnwcI3rFSS9bmY74WkuD8KfdWYULTCzLYBfAo/iK4ElzWwvlXxkhqqdBCwt6dPR/g+DIj0JZraRpGvNE2YVZXGVC6L/TQnSjseV4HF98FX4R4EvpdLUhMwoBoAwAyhjLjzpyH0VxyahYCYpssVod3f843lcOOdW+uTaTbOljmOSbrAKG/MmeicXWmh5ryIT2Oicm8zspFLxaDM7Fg+hDC6qGsPwYPrwIm+Nh5v/r4UMfjHCynMBOh21hl0cOMw4Bvd/KUJ0LI17NHf7ge9VO234t5mtp74UAOvSP0xJE9bHY3F9vOKY6Pzwx9npnsEjGBTZ6daQtCclSPqTmf2oC3oakRnFwDCGPosbwv+X8GiWX4IkZjI6paPwUflxg3ioCJPQ71RKYRIssjHHQxcvjM++empjnoIWWpoU1eVV0n746uQc/DlcTTcpHnuLX+GWK3fjuZAXx62cJsHM9sNl28/TFwtJuFnktIw31Bld4DH64pkNRztt+BJwRtBVGJ786nOpJ0s6OPymGDKY6j2z++VkjzAi6DBfk3RKR4Nme+Aiq5+m0Jt1FJMJZjaRembyI3y2nyRbtATPzESaxjEENuaDpcXMLsU92C8rnbM5Hu5i87C/oDy21Xtime6UBDObTtLb0f4E4IPqcUiJdzvCSnFx4Fz8XdkWj5/1FwBJZdHKZG2nC7pHhXYHZLSQohQ3s4fxCcg5eNSGV6NjN+D5XToSlZnZB/DV1ax4RIByRIMZ8FDpSROUvKIYAMzs/5qOS7pRLdEazawICpYiW3wMuCXoNzo8M4PI5r/FQDCz5fAQEo/LQ43HGCob8xQ00fIV4FIz244+MdKaeFTOLaM2DjGzb+GOReXwyUOOYKX1aTxkQ/xuHRL9fwrXR2V0YiZ8lVUYgbwIzIyLZsqimKFopxJmtrOk31lnjLZJ+gUlpOAt4VRcKV6EAf8snvRqku5C0rLmseM+A3zHPN3r2cF68hvAueYBSuN3ZZdQ/+QykwhtvmXWP9Z7HTKjGBi+UVFWiA8WBUa2MRO6ky0+GrYR9BfLXIGbAz5iZu/Fc+GeBWxpHr32oKjuDTYENuaJqKVF0iNm9n5cab1SUR+3Hf8PQDALfBK3mjrVzHZRKf3nMOBCnAmMIfLWLeEx4Pqwaoo9erv9wExVSBTBDFk7DZg1/FaJRwcy6UpSiocVwx1m9mM8TtgZwO/kqQXWwsWtnwvV78VXrS+Yh1tfQFF8OQAzW6AbIrPoqQcIiqzv4jqIwyRdbO5HUUbMTB6W9L6a9h6oOmYVnpmx6MjMDsVj3OwTlpZjYrGSDZGNeQoGS0uQ/6+HW4D9HrhFPc6L3C0s2La31GkNxzAtwvqcStcNRZOcSoejnYR+1lUp8mpVWUI7f8VFR7FS/GhJ60R1RgGfxFcISwN/As6VVGm0YWarKyTCMrNd8NSpX6PTM/snuMFFmp+FhiFF4tSy4YrX63G9wyYtddfFLS9uw5fBNwBrVdT7AHBjqWwd3JvyybC/CnBi+B+nYrwF2Drar82JiyuRVx7uezgYWnDl94jiXgz3hucYeH9i3dlwh7xhp3tK2HAjhN1wKcd0+Oz46uFqJ6Gffqlyq8oS2lkFN354HA85Pxb3x4jrTMQtHtcZCG24Q+4NuI70pfB/827ozCuKASDYan8HFzMcpjAbqKm7MW6VI9x66epQvhaucDudCtmiooRH1uyZ+Ts8DMQzuN3+kvJQ1nPiOahXidq5Hs8pPF3o8wU8MFhdtrDJhsHSUlYSTwkIsuP34i/2m9A/raV5fpHf4swR3Ft/F3UmtZnmYA1OpcPRTkP76+DBHb9CZ6ykUcAn4/ety3ZrleJmHrysi7bGqscOp1lHMTBcDDyNc+cDzXMgTILcuzhmJt8tMxO1yBbLHUp6qqR7KmLgfwF3wFkCDzBWiKZWAI4uNTOH3CHs83gylIPNo5oOB5JoMc+jvZii4IABd+BJnjCzEyTtN/lJbkVKDoyTgQMkXQdgHkr91/jHZ1pGk1PpcLRThxnw1eB0dOopXscnc12hbAARKcUPMbOfSvoKcJFV+OOo3mS+Q4xpnh55X9xbHdxZ7+eSrk+lMzOKgSElIFgrMwkMYZLMOsgWq9IT1npmyqOTHhFXNo9n/1dJt5bamc7MFsQtLL6TcA2TE620mGeFOxp/OZc0T7ZySHhBYq65bsXpQw5JT5gHbPtwKLpJ0t2larMWTCKcc715VrZpHU1OpcPRTiXUF7jzdPVGJ9ZkAFHEditP+NqwFp4KuZB+/By3vDsEf29Wxw1A9lXJBL0OmVEMAKoJz1HCQKJL/oYwSy6h1jPTPAf3EbjDz6H44JoXd7bZRdIVUTuH4ErjmyXdaR488JEB0NkLpNDyA3zQXw8gaZx5vmAYPrPeWpjZ/vgKrzDB/J2ZnSzphKjaY2b2PToDPD42hGROcbB2p9IhbScR/zKznzD4PDKLSNqs6oCCsrrpe2Nmx5eLgM+a5w0HN57ZujRhGWdmo3GGmsQohl2J9W7c8Ly9+0T7t+Mv+2PANoNod2xp/8jwu23DOaNxy6FtgVdw5xrwZebYgdIyJWzAbeX7QlDe4wmA7gHGR/+L/XuGks6YNnzFUOzPWqYFt4w7HrdAuQufAMw13Pd6uDfgZmCGKaWdhH6uwq32HsB9Nk4t3tcu22k1gMBXzFcDD4dvzETgsXDsKTwM/y54Lptdcd+R4v+DDe3WHutXd7gHyLtxw62LFo32xwHzAIsB14SyrpkJkcVS2B+PzxBqrSmAcdH/B0rHxpb2j8KVbtMD14QBtfMw3cNWWvAY/DuGD/Ay+Azol+HY4k3bMF3TeGCmaH8mYPxwj9d3wwaciUfc/R5wQLENVzsJ/YwJv7HV4Z0DaOd+PErDQ9RMdIAHcf3X/OE7Mw+exwZcT/JT3ER8oVD2WJnOpmtI2bLoaWCYQdJT0f7N8pAML0Xy5gNxu+cCM+Kmr7Pinpfnm2efQtJz5qE7RpjZiuqzgLkCXyXMZh7PqcjmFqdU/F/URzkoWVk805pNawiRQst+uP7iTfxFuBIPf4KG2WeiBqcBt5vZn8L+1jizo1BMWv88JcDwhHqfwtDkVDoc7bSh8HZ+NugB/kafJVs3SDGAeE010W8lvQF8xczWAM4KjpxxROWlzSM6lGF4RsgkZPPYAcDMJkh6b82xR+VJ3e+U9IGo/OeS9g3/b8M/KgfhD+xI3PLpXtyJ7Ch51rsZJb1pZhdK2qqmv3fwsB6GhyoorJ4Mn91OH9UtTGp/gydHusJCXPvB3I+BIIWW2HHo3QIzW52+REs3SRobyteQNMaq85SgNL1XxhQCS8gj03L+KLnVXyVzkfRyVLc2P3epTcMjHKwjaedQVjneojaSxl1mFAOAmZ0FXC/p16XyvfDsaju0MRP8g/5B/OP+BB5a+zkzmwvPgreqmd0laXUz+62kz/aA7tZsWkOFFFrM7DrgPcD5wDmS7h1qOrtFeH6L0hlC/K7o+P6SflY6p1/ZtIbwrKtWWl0ph3vVzuSGmV0iaUvrHzwUXFqwVFT3un4NeJ3kazJPcFR8jyaoy4RGmVEMAGY2P25+9iadbvEz4nqG59uYCbCcpMIPoDyTHitpNTO7F/gxbs3UL76UBhAJ0zqzac2KR7AdlsirKbQE8dx2wPb4rO0cST8q1anztRhSmIdQ+Rwu+iherI4XumD+pfPGahgy8k1JCKKTAjPhvgVvSzqw5pTJ2k5CP2fgoUFeDftzAcdI2r2X/QwW5sE2f4ybDT+BM6RFcYnGd1QRMLCyncwoBg4z2wg3jwO4T9K10bFGZoKbpa0tT26ziEIsmsD5b5enOVwP2An/UJaXtOp2UJpHmj0A/6juaWbL4AzrkpZTe45uaTEPEnggsL2kGaLySb4Wksq+FkMKM3sIt2DpFw7ezHbAFfPr4SKLArMD/5M05DlBpnSY2R2S1ppS2im12Y+5d8Pwg4iyFpLuslKEWnzy8XdcJzoxsZ/j8DH21aDPKLzAjwb+LWn/lHayMnsQCIzh2ppjLwAfKjGTSwtmEpS4CnXjgGXz4AG8kHtz32xmoxUlHgkz6DjcdipOwx17Ci/gZ4DzgCFnFCm0mNn78JXEp3HHxXMI9ybCD6j3tRhq3IuL0KqcJm8FnsV9XI6Jyt/ArV2maZRk9SPwSdUcw9VOAkaY2VySXon67eZ7ekzDMQEbUa2MXwIPNf4DSWfHB4JBDOrMcbMlsKyiFUHQjXwJt6bKjGJKQB0zUU3qS0nP4B/NuOyU4Ey0KR6SYBPcXvy8LslZWtL2YXaLPCZUckz6HiOFllNx5rCppL/VtPNfSa+VTh2uZfLhwNggMoyVjp+QW2k9gQd4zOiPOGvk27ivwB7D2E4bjgH+ambnhb62AQ5LPVlSq0OuaiIKB6b0F+Ds8M4cjIfoGOGH7W3gBEmHeDP9xUZB3Jv8nmRGMYUjWC3siCcjugN3vllKpXDjiXgrrEYU2l6a+rwJkxuttCgKtdyA+8xsRzwHyDJ4SOVy6JKhwhm4Bdt4Os2WJ8HM3qCPkc2A+5H8U27qPM1CLYm+hrqdhH7ONPduLvRPn5J0f+r5ZvappuNN+kdJL0eTqq/i34QPFOIo8ygHJ5nZV4H7rSJXi3k8rAeT6c06iikXZvY0npznJODPkt4ws4kDfRnMEwR9Fw8YeBU+wD6nLoKD9QpNtJjZuZK2M7PxdK4OqqKxzoL7WsR5LQ7t1qqjF7CSSXRCfcMdM9dWZ4KpaQZmdqCko8L/bSWdFx37saRvD2U7XdC9WFV5naSg4vzTGg436h/Ng/x9T9JGZjYWT3Hw91Kd+fD3akvcrPbfdEapnhmPdtshvajtMzOK4UeNbBEz+ymu+L4Xdzi7EPf0TXaUqehrHmBt/KN6Gy4Cur35rMmDOlqsLxf24lXnacp0tsPMjsVXRRfRYO9ecd40a/UUW4GVLcKqLMQmdztd0B1PYmYGlgQekrRi/VmD6qPA3Lhz3y6SHrSGZFnxsZKu9H5J13RDSxY9DRMSZIvIPXm/ipvT7oCHvZjDPJf0ZZL+0W2/cg/ySyM67sJDjww56miR9Gwo2lvSN+NzzOxI4JvR/rLA1ynlqdbw2M0XH/u1o7JCMQn0EzmMwGd3Q776mYJgNf+r9oeinSQoyhwJk6yY9k49v8Kiqdz+sfQ3WBHwkqR/RmX9LOyqjjUZ3qQgM4rhQ6NsUdJx4GtQPIPedeYhxguF9om4Bc1gMVzK7CqUadmEiCkEbF4qOw/PdPcb+nJ0DAtSFJR4dsMCb+PhSyq97qcRqOZ/1f5QtDMgBHPWbhxXW8OLJK6cVzEP71OGEUW1HSyy6GmY0CZbbBNFmNnM8lwUg6XjSUnDsqIoo6AlmO7tjceieTSqMjueG3vn6JwxktZgCoB5VsFd6L+6+fIwkTTFw7oIQTMU7XRBd7wiGIGnB5hH0qa97GdKQV5RDB+mLzMJcD1FWDk0ohsmYTWB6PCXaJ7UdnqBRFp+j+cXPxyPh1XgDYUYOJG9/MVmtjeecD7WC7zM0OMyXNfSZPW0FB5afG38PvwVd4aaJnNSSBo5JbXTBeIVwdu4CPWPQ0xDYczxXwUPazNbDreQfFzSnxpP7qafvKIYHjQp2HqtfLMeBQYbLlrMvdzj5DBPWnWMnKjKwBX+A0XKczMPCPkL+lJ1fgbYT8MQbyvj3Q8zuxHYQ9IjZvZe3IT+LNya8M5eWdNlRjFMiJbK/Q4xGZbK70aYh+c4FlgI93ZeHM+5saKZrSPpr8NKYAnB8OAfuHd55erGzO6JzXtD2bBE8M3oHg0rYiA9XLyFQJBmtq6kWwZBz/hCsW4ea2xuSfuY2Qx4von3N7eQhix6GiakLJUrBmUR6+U6ScORQ2Ko8SNcRPMXeZDEDfHUoeCz8p6aPPYAbwE/wf06JgUFBJaKRGWXm9lBwNnh2PakpqPMmBJQ5K/+FB7ZuHgPdwCe76Kd3XAR5AkMbhzH34eN8PGHpLfMrFL8ORDkFcUwIUW2WCOmmRv/WD4ytTtpmce4WtPM7gZWk/S/YvY9JfoemNljwFpVuqcpUVSWMXAUY7OtrOH8P+Cm0QvRabDRz6m0pZ3fAc/hYX8OApaUh8OZE7ihVyvVvKIYPlyBx6ApZIt/xWWLW5rZByUdVKc7MM9YNYZORe/UiFfNk8TfhGfveoE+cd2SVp25Cxi2jHET6LO26YCGKLRExpBhVjNbqjBCMA9EOWvLOZMgz1nzHjySwGDG6hfwwH5L4Fkji/G3An2rn0EjryiGCYOVLZrZOEmrJvY1xYiwuqHFPEfFv3Hzw53wKKBnSXrJzB4BPl/Xz1Aq6AuYp0BdEfd7iXUUXzazjSRdazUxfjSA3CIZwwcz2ww4GXgMXwUsDuwl6coBtDUDsGzYfUiJOSIa2psXd8zr2cc9ryiGD62yRatOkzgXbqt/X8WxOlTNLOYGdjazlYZYhJVMi6R/hjAey0g6I4jrCt3OG8PBDFrw57BVYX3cM/bjFceEx+PJeJdAnrp3GWD5UPSgpK4DbAbx8pm446UBi5rZrpJuTDx/beAI4GU8wdlvcUfcEebBAK/olqZKSMrbMGy4Euxo3EP7eWCWUD4ncHf4PxGfsUyM/t+Jh/IY1QMaRgLjhvte1NGCL6vvBB4N+8sA14T/Fww3zTXXMQOwUtimLx0bAWw33DTmbVDP98Do/7alYz8eQHtj8IRdxf6yuEQh9fzReEDMbYFX8ACT4AxsbK+uO4uehgnmIbb3BxYETpV0dyj/EB4c77dDRMc4JYqwJjfKtJjZODwp0e0KiutYZDelwcw2wEONPw6TUk7uqmh22I3CM2PKg/U4+GCNuXS/sobzJ70zZvaApPdFx8aqRwYfWfQ0TJB7Vh8RlwXZ4l8l3Rr2/6+ljdTlaa9EWINGl7S8KRfFFedOx/AlJUrBMbhC8SGYFLDwD3iWtQJ/MbOv4wmZJvnRaHg8yTO6R6+DD442s9/QZ2a7E75KSEVsAluO1pB1FO92JMoWv1FxqoCV8dlqatiCOOtX0cZLuNL1SwO9hgGiG1puMLNvAzOb56/YG7h4qAgdAKYvmASApIcrwrFsH373icqEx7XKmPLR6+CDX8LHQhEP7CY84GcqiqCAhr8nRYDAHBRwaoB5dqxv45Y8JwObS7rNzJYH/lC1ZDSzdfFkP3MBh0makj+ag4b5UuLzdCYl+o2iQWtm10jauHRev7KhgJmdis/witnhzsAIRUlozGwmlZIqVZVlTJmwIQ4+OKUgM4phQjeyRTPbGPgePmP5saSru+yrJyKsXiCVFvMc4fdJWr6qnpnNBMyCr0Q2oG+FMgq4ou68yQkzmxGfHa4Xim4ETlJkDVMlxx6IbDsjYyiRRU/Dh1bZopltgYeDeA34rqSbB9hXr0RYvUASLfLk7w+Z2WKqTi+5F/AV3LN1DH2M4nXg570mugnmoeHnk+dMPjZsmNmKOON6MThXLYyLB1ajk7HNMpT0ZmR0i7yiGCakLGGDP8XTwN1UyD81QO/jKUmE1USLeWTM1fCImLHi9xNRnf0knTB0FPeHmZ0NnFhemZnZh4EvSdrRzHYFPoeHbbiTTsZ2hrLD3TQJM3u/pPHDTUcbMqOYglET62kS1KXD2WBFWL1ECi1111++bjNbCQ9ZEIciP7OnBDegyeTVSjmNzexASUeV6iypkOUwY9qCmd0EzAicjkcdeG14KapGZhTTAEoirMMGIcIaElpC7KsFVAq/bGbrAc9KejQqOxjXUayAR2HdHLhZ0jaT7SL60/uQpOVSjtXoKKaYLH0ZQ4/g4b077jR3B3DacE7iqpAZxRQMM9sKWETSL8L+7cB84fCBks5PbGeyiLAGghRazOwS4FvlJbmZvR9ffXw8KhsPrIJ7oa5iZgsAv5O0yeS8jhJdlwK/kHRZqXxz4MuSNg/WbCviXvWxnmYU8A1JKw4VvRlTHoLxxtbA8bg40uD/27v3YLvq8ozj34dgICjXclFRINyEci0Xy60tSKODFQamMyCBljoDOELlUlumop0ySoc6Kg0TpxSh00ILFloNV0FLwXAHE25JwACWiw6xIJYYKASMT//4rQP77Oyz905ystY6nOczc2ayfnvtw+uMZ797/S7vy7ltmZLMYna7nUPpgDZiPWB/SpXKfwKGShTAYeMc15oYJpates3b2l4gabuu4ddcyo//StJGlAZHHxyHOFfFWcBNko6lLKxDWYs4EPhEdf2h6t+bMLre0zL6FDeMdzZJe1J6U/wB8J/AkbYflPR+SkXpJIoYaKrtn3Rc32X7JeAllcqqQ1nVtYy1achYNunz2rSu63kqtfcvpXxIv0L5A6uNSxvKPYCZlBpPAHMp1URfr+65DrhOXZ35qm2+vQoFxuQwG7iM8vTw1u5H289L+mJzYY2WqacWk/SU7R3HeO3HtncY8veMyxTWeBgmFpWmLrfZvrTrvScDM2wfRw/V08ZGth9da/8DxkE1zfAxSle0GdS8phLtIeks27O6xs60fVFDIfWURNFikq4EftDjA/PTwKG2jx/y99wNfHLk6aQqtnc41RRWnaeYh4mlWmeYQ2kt2jmVMxU4xvbPun7n1pR+AG89Idd5iHBY1S6umZROhg8ABwPb++1mMzHJjLG5YdSB2zbI1FO7nQ1cK2km8GA1ti9lreLoVfg94zKFNU4GxmL7f4CDVHpkj0zl3GT7tu5fJukrlPpJjwErqmFTTkW3hqSfAs8BFwN/bnuZpKeTJCYnScdTvjRM1+hOjRtS6r+1Sp4oJgBJH6HsmIFS1mKlD8wB7x+XKazxMN6xSFoM7OnVaBqzNqiUj9/GHcUBq/FZlOS+ELgKuA5Y4PTKnpRUGnJNBy5gdEvjZcCjtn/VSGBjSKKYBMZrCquNsUi6mdJA5pVxDHO1SDqS0oxqqu3pkvYGvjSy/ViSKGc+jqdMP21M6Zv+3TbEHzGWJIpJQNKWlBady+kxhVVN9UyoWCTNpkwxbU05R/FfdPWpHr+ohyNpPqWt7Q88oNGSSvnxkQXtj9nevNZgo1GS7rJ9iKRljD5PJMC2N2ootJ6SKCaRNZ3CalMsVe2kMdm+fHVjW12S7rN9QOdipIboViZpWufWyIi2yWL2JFJ9GDeWHDqNQyxLgXtsvzBOIY2HRdXGgylVWYYzgHsGvSlJYvKStAPwU9vLVVrp7glcYfvlJuPqtk7TAUSsphOBhyQ9KelySadWxQGb9FnKU9JyyoL1Usqp7YixfBtYUdU3+yalqsBVzYa0skw9xYRWHbI7qPo5ENgG+KHtjzcQyz62Hxx851v3rwO8x/YvB94c70gj5ygk/QXwuu3ZbTxHkSeKmNBsP0NZFH8IeJhS66m7zEddvi7pcUlfHuvpRtJVkjaqzowsBB6rPiRicnqzOlNxEnBjNda6dqpJFDEhSTpX0g2S7gM+Tzm1/Q3KmYpGiiBW/93DgBeBSyQt6FGv5zerJ4ijgZspe+n/qNZAo00+RXkS/hvbT0uaDvxLwzGtJFNPMSFJ+hGl690NlAXj+92ipi9VkcBzgONsT+0YXwTsTZmH/obtuZIesb1XM5FGU6qaX1fYPqHpWAbJE0VMSLZ3oRTUm0c5xDZH0gOSLpX0qSZikrSrpPOqHhmzKQnsA123XQI8Q6ltdUd1QjdrFJOQ7RXAtpKmDry5YXmiiAlP0rqUQ3u/C3wamG57SgNx3AtcDVxj+/lVeN+6bSvZEPWQdAWwK3A9o/vCX9hYUD3kHEVMSJKOoux0OpiyJXURcDfwOYY4u7A22D5w0D2S1gP+ENiO0X9/X1pLYUW7/bj6WYdSELCV8kQRE5Kk71ASwz3AfNtvNBjLNbaPraacepVj2LPj3lso5yvm83a1W2x/va54o30kbdDmSsJJFBFrSNL7bC+p1htWYvvZjnsX2m76YGC0hKQDgX+knKfZRtJelM6IpzUc2ihZzI5YQ7aXVP88zfaznT9A9x/8PdWOqAiAWZTikC8B2H6EstbWKkkUEeNnRo+xI7quDwHmS1os6dHqrEWrW7fG2tXVyAs6piTbIovZMeFV2wt3ri4X236z5v/+ZyhPDtt3fehvSFlH6dSdOGJy+4mkgwBXpefPBB5vOKaVZI0iJrSq4ubllLMJohRVO6nOntmSNgY2pUe3MtsrtbWs5qF/p7q8s5puiElI0ubARcDvU/7/+33gzKo9cGskUcSEVjULmjnSelTSzsC3bO/bYExbAuuPXNt+ruO1M4FTgO9UQ8cA37Q9u9YgoxUkbWH7xabjGCSJIia0Xo2BhmkWtJZiORK4EHg/pTjhtsDjtnfruOdR4EDbr1bX7wbubSLeaJ6kJyhPw1cD325bH4oRWcyOiW6epMskHVr9XEop69GE84EDgCdsTwcOB+7rukeMXqxcUY3FJGR7Z+CLlEOjD0q6UdKJDYe1kjxRxIRWnXQ+nbKbCOBO4O9tLx/7XWstlnm295P0CPBbtn/dXfBP0p9RSkrPqYaOBv7Z9qy64412qdYrLgROaKIETT9JFDHhSdoCoOm5Xkm3Uj74/xb4Dcr00/62D+q6bx86Epvth+qMM9pD0kaUdapPAjtQvkBcY3t+o4F1SaKICUmSgL8G/pS3p1BXALNtN1I3qVpveK2K5wRgY+DK7h0skjal7M56a3v6qnTGi3cOSU8D11KSw70NhzOmJIqYkKopnCOAU20/XY1tD1wM3GL77xqKa1tgJ9u3StoAmGJ7WcfrXwb+hFIIbuSPz7Y/Unuw0ThJ8gT4EE6iiAlJ0kPADNs/7xrfAvh+Ez2HJZ0CnApsZnsHSTsB/2D78I57FgN7NFnEMJonaZbtsyTdwOhCkgDYPqqBsMaUk9kxUb2rO0lAWaeoTrg24XTgw8D9VSxPVmcqOi0ENqGsX8TkNdLu9GuNRjGkJIqYqPp9I2/q2/py22+U5ZO3Gip1f1u8AHhI0kLgrZ1ZbfsGGWvXyGK17blNxzKMJIqYqPaS1KuFqOg4FV2zuZLOBaZJmkGp/3RD1z2XA18BFgC/rjm+aBlJBwPnUQ5nrsvbPUy2bzKublmjiBgn1U6sk4GPUv7gvwdc1rlYKemHtvdvKMRoGUk/As5m5UZWqfUU8U4jaQqwyPYuA+67kDLldD2jp56yPXYSknS/7d9uOo5BMvUUMQ5sr6h6TGzTWQSwh5HdWAd0vh3I9tjJ6XZJX6UUiWztF4c8UUSME0l3UBLBA8CrI+NZqI6xSLq9x3DrztUkUUSME0m/12u8c2eLpE2APwa2Y/TJ7DPWcngRqy1TTxFrSNKOwFbdWx0lHQIs6br9u5SKstn1NIlVlQU6Gfg5cNdIpYE2SaKIWHOzgM/3GF9avXZkx9j6trs/JGLy2bDH2HbAFySdZ/vfao6nr0w9RayhflteJS2wvUfH9dnAK8CNjF68XKllakw+kjYDbrW9T9OxdMoTRcSa26TPa9O6rt8Avgp8gY6igECrDlhFM2z/QiNH+1skiSJizc2TdIrtSzsHJZ1MOUjV6XPAjr3qVEVIOgz436bj6JZEEbHmzgLmSDqBtxPDfsBUSlOaTk8B/1dfaNFGkhawch2wzYDnKbviWiVrFBHjpPo2uHt1ucj2bT3umUPpj3w7o9cosj12Eqn6lnQy8JLtV3vd37QkiogaSTqp17jty+uOJWJYSRQRNZM0Fdi5ulxs+80m44kYJIkiokaSDqWUGn+GUmH2g8BJtu9oLqqI/pIoImokaT4w0/bi6npn4Fu29202soixrdN0ABGTzLtGkgSA7SeAplq3Rgwl22Mj6jVP0mXAv1bXJwLzGownYqBMPUXUSNJ6wOnAIdXQHcDFtpeP/a6IZiVRRNRA0hbAFrYf6xrfDXjB9ovNRBYxWNYoIuoxG9i8x/hmwEU1xxKxSvJEEVEDSfNs7zfGawtt797rtYg2yBNFRD169R8YkV1P0WpJFBH1eErSx7sHJR0B/HcD8UQMLVNPETWQtBNwE3APoyvMHgh8ojpPEdFKSRQRNam2xs6ko8IscJXt15uLKmKwJIqIiOgraxQREdFXEkVERPSVRBFRM0nTJH2o6TgihpVEEVEjSUcCDwO3VNd7S7q+0aAiBkiiiKjXecCHgZcBbD8MTG8unIjBkigi6vWm7aVdY9l6GK2WfhQR9VokaSYwpTqEdwblEF5Ea+WJIqJenwV2A5YDVwFLgbOaDChikBy4i6iRpH1sP9h0HBGrIokiokaSbgfeC/wHcLXthQ2HFDFQEkVEzSS9FzgWOA7YiJIwzm82qoixJVFENETSHsA5wHG2pzYdT8RYspgdUSNJu0o6T9ICSnvUe4APNBxWRF95ooiokaR7gauBa2w/33Q8EcNIooiIiL5y4C6iBpKusX1sNeXU+e1MgG3v2VBoEQPliSKiBpLeZ3uJpG17vW772bpjihhWFrMjamB7SfXP02w/2/kDnNZkbBGDJFFE1GtGj7Ejao8iYhVkjSKiBpI+Q3ly2F7Sox0vbQjc3UxUEcPJGkVEDSRtDGwKXAD8ZcdLy2z/opmoIoaTRBHRAElbAuuPXNt+rsFwIvrKGkVEjSQdKelJ4GlgLvAMcHOjQUUMkEQRUa/zgQOAJ2xPBw4H7ms2pIj+kigi6vWm7ZeAdSStY/t2YL+mg4roJ7ueIur1sqT3AHcCV0p6AXi14Zgi+spidkSNJL0beI3yNH8CsDFwZfWUEdFKSRQRNavKeOxk+1ZJGwBTbC9rOq6IsWSNIqJGkk6htEG9pBraGri2sYAihpBEEVGv04GDgV8C2H4S2LLRiCIGSKKIqNdy22+MXEhal9FlxyNaJ4kiol5zJZ0LTJM0A/h34IaGY4roK4vZETWSJOBk4KOUpkXfAy5z/hCjxZIoImoiaQqwyPYuTccSsSoy9RRRE9srgMWStmk6lohVkZPZEfXaFFgk6QE6TmTbPqq5kCL6S6KIqNdfNR1AxKpKooiogaQdga1sz+0aPwRY0vtdEe2QNYqIesyiOmTXZWn1WkRrJVFE1GMr2wu6B6ux7eoPJ2J4SRQR9dikz2vT6goiYnUkUUTUY15VEHAUSScD8xuIJ2JoOXAXUQNJWwFzgDd4OzHsB0wFjrH9s6ZiixgkiSKiRpIOA3avLhfZvq3JeCKGkUQRERF9ZY0iIiL6SqKIiIi+kigiIqKvJIqIiOjr/wHYRuyWwbHeFgAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "our_repos = pd.read_csv(data_path/\"repo_infos.csv\", parse_dates=True)\n", + "\n", + "# display bar chart of the most popular licenses\n", + "our_repos.license.value_counts().plot(kind=\"bar\")" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "our_repos = pd.read_csv(data_path/\"repo_infos.csv\", parse_dates=True)\n", + "\n", + "# number of our repos with an MIT or Apache license\n", + "mit_apache_repos = our_repos[(our_repos.license == \"MIT License\") | (our_repos.license == \"Apache License 2.0\")]" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([ 19163.2 , 48317.6 , 218033.08])" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "np.percentile(mit_apache_repos[\"size\"].values, [90, 95, 99])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(357429, 339557)" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(mit_apache_repos), len(mit_apache_repos[mit_apache_repos[\"size\"] < 48317.6])" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "mit_apache_repos_filtered = mit_apache_repos[mit_apache_repos[\"size\"] < 48317.6]" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([ 19163.2 , 48317.6 , 218033.08])" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -85,14 +242,14 @@ "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "(544340, 517122)" ] }, + "execution_count": 22, "metadata": {}, - "execution_count": 22 + "output_type": "execute_result" } ], "source": [ @@ -110,20 +267,9 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 22, "metadata": {}, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "648023" - ] - }, - "metadata": {}, - "execution_count": 24 - } - ], + "outputs": [], "source": [ "# Combine our repos and EleutherAI's repos\n", "def combine_repos(ours, eleuthers):\n", @@ -136,8 +282,17 @@ " dedup_combined = combined[~combined[\"name\"].duplicated(keep=\"last\")]\n", " return dedup_combined\n", "\n", - "combined = combine_repos(our_filtered_repos, eleuther_repos)\n", - "len(combined)" + "# combined = combine_repos(our_filtered_repos, eleuther_repos)\n", + "# len(combined)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "mit_apache_combined = combine_repos" ] }, { @@ -155,10 +310,11 @@ "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stderr", + "output_type": "stream", "text": [ - "/home/nathan/.local/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3441: DtypeWarning: Columns (1,3,4,6,7,12,13,14,15,16,17,18,19,20,21,22,23,24,26) have mixed types.Specify dtype option on import or set low_memory=False.\n exec(code_obj, self.user_global_ns, self.user_ns)\n" + "/home/nathan/.local/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3441: DtypeWarning: Columns (1,3,4,6,7,12,13,14,15,16,17,18,19,20,21,22,23,24,26) have mixed types.Specify dtype option on import or set low_memory=False.\n", + " exec(code_obj, self.user_global_ns, self.user_ns)\n" ] } ], @@ -167,11 +323,11 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "Shard the dataset into manageable pieces since EleutherAI's downloader has a memory leak." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -202,8 +358,175 @@ "metadata": {}, "outputs": [ { - "output_type": "execute_result", "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
namefork projectcommitsbranchesdefault branchreleasescontributorslicensewatchersstargazers...total issuesopen issuestotal pull requestsopen pull requestslast commitlast commit SHAhas wikiis archivedlanguageslabels
00-1-0/lightblue-0.4False8.01master0.04GNU General Public License v3.014.086...98502020-10-18 21:26:07.09a4f7b37e923b262d2a29894676ff8ed8cde6237TrueFalseNaNNaN
10-14n/ndroidFalse131.01master0.02Other5.050...11212015-03-17 13:10:07.04e5dbe69855a7fda8b74e61d9db5aa61e6ba9ee8TrueFalseC,C++,Objective-C,Shell,Assembly,Haxe,Groff,Py...bug,duplicate,enhancement,help wanted,invalid,...
30-sec/zero-crackFalse4.01main1.01GNU General Public License v3.00.062...00002021-05-12 02:03:08.070ee16550a81b396333565515723d5abab87c719TrueFalsePythonbug,documentation,duplicate,enhancement,good f...
40-tikaro/minimum-viable-startpageFalse15.01master0.0?MIT License4.056...00212019-04-21 09:11:12.0a4fb4aea4474d635c4e4738f7d8c1a485d5d74c8TrueFalseJavaScript,CSS,HTMLbug,duplicate,enhancement,good first issue,hel...
50-u-0/dugon-media-serverFalse52.01master5.0?MIT License2.014...51002020-05-16 04:11:45.01d6bb1c589e51d2c34b11be20d34dae4bb0c7779TrueFalseJavaScript,Dockerfilebug,documentation,duplicate,enhancement,featur...
\n", + "

5 rows × 27 columns

\n", + "
" + ], "text/plain": [ " name fork project commits branches \\\n", "0 0-1-0/lightblue-0.4 False 8.0 1 \n", @@ -255,11 +578,11 @@ "5 bug,documentation,duplicate,enhancement,featur... \n", "\n", "[5 rows x 27 columns]" - ], - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
namefork projectcommitsbranchesdefault branchreleasescontributorslicensewatchersstargazers...total issuesopen issuestotal pull requestsopen pull requestslast commitlast commit SHAhas wikiis archivedlanguageslabels
00-1-0/lightblue-0.4False8.01master0.04GNU General Public License v3.014.086...98502020-10-18 21:26:07.09a4f7b37e923b262d2a29894676ff8ed8cde6237TrueFalseNaNNaN
10-14n/ndroidFalse131.01master0.02Other5.050...11212015-03-17 13:10:07.04e5dbe69855a7fda8b74e61d9db5aa61e6ba9ee8TrueFalseC,C++,Objective-C,Shell,Assembly,Haxe,Groff,Py...bug,duplicate,enhancement,help wanted,invalid,...
30-sec/zero-crackFalse4.01main1.01GNU General Public License v3.00.062...00002021-05-12 02:03:08.070ee16550a81b396333565515723d5abab87c719TrueFalsePythonbug,documentation,duplicate,enhancement,good f...
40-tikaro/minimum-viable-startpageFalse15.01master0.0?MIT License4.056...00212019-04-21 09:11:12.0a4fb4aea4474d635c4e4738f7d8c1a485d5d74c8TrueFalseJavaScript,CSS,HTMLbug,duplicate,enhancement,good first issue,hel...
50-u-0/dugon-media-serverFalse52.01master5.0?MIT License2.014...51002020-05-16 04:11:45.01d6bb1c589e51d2c34b11be20d34dae4bb0c7779TrueFalseJavaScript,Dockerfilebug,documentation,duplicate,enhancement,featur...
\n

5 rows × 27 columns

\n
" + ] }, + "execution_count": 45, "metadata": {}, - "execution_count": 45 + "output_type": "execute_result" } ], "source": [ @@ -469,8 +792,8 @@ "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" }, "kernelspec": { - "name": "python3", - "display_name": "Python 3.8.10 64-bit" + "display_name": "Python 3.8.10 64-bit", + "name": "python3" }, "language_info": { "codemirror_mode": { diff --git a/scripts/get_license_info.py b/scripts/get_license_info.py new file mode 100644 index 0000000..6c1b266 --- /dev/null +++ b/scripts/get_license_info.py @@ -0,0 +1,31 @@ +import os + +import pandas as pd + +from fastcore.script import * +from ghapi.all import GhApi + +GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN") + + +# Open issue on repo using custom title and body +def get_license_info(owner, repo): + api = GhApi(owner=owner, repo=repo, token=GITHUB_TOKEN) + license = api.licenses.get_for_repo(owner=owner, repo=repo) + return license.license.name + +@call_parse +def main(repos_path: Param("Path to the csv containing all of the repos", str)): + """ + Use pandas dataframe from the repos path to open issues in each of them. + """ + repos_path = Path(repos_path) + df = pd.read_csv(repos_path) + + # Loop through repos and get their license + licenses = [] + for _, row in df.iterrows(): + owner, repo = row["name"].split("/") + licenses.append(get_license_info(owner, repo)) + df["license"] = licenses + df.to_csv(repos_path.parent/f"{repos_path.stem}_with_license.csv", index=False) From 98fecd06b99b9ae94dae8b1c555603913e8e61d8 Mon Sep 17 00:00:00 2001 From: reshinthadithyan Date: Sat, 10 Jul 2021 23:36:19 +0530 Subject: [PATCH 05/15] add parse check and tree-sitter --- .DS_Store | Bin 0 -> 8196 bytes metrics/.DS_Store | Bin 0 -> 6148 bytes metrics/parse_check.py | 53 ++++++++++++++++++++++++++++ metrics/tree_sitter_utils/.DS_Store | Bin 0 -> 6148 bytes 4 files changed, 53 insertions(+) create mode 100644 .DS_Store create mode 100644 metrics/.DS_Store create mode 100644 metrics/parse_check.py create mode 100644 metrics/tree_sitter_utils/.DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..1278349e2a33a230bdf03ef2c278632821a45f48 GIT binary patch literal 8196 zcmeHM-A)rh6g~qeZ4J;6!cU^f-slw&3K--xJ=WCYd?Y+4G$}^UayrJzF9Y>-9p0Xqt!;s4UYL z&~zz$oolQN=#eQ%0X&gIWwPluq#M&V1BL;^fMLKeU>GnA{0R);naw594A*vW5~z5RUI%fm0;%4?|e-(*3LXq5J~FE)B*iDTqE<0IJj5$uSZLjm)u zO5{G~5x@fc4hIan$J zIVWdPfjpwc-aLq}zhrT#x5a^jh{eet`q*=o20lh?jbWB53hI={+&(O{{ z{Hl)F5BX}jAZ7tGD4_gT7kgbWBU%Vyw z2y(!yI*hyc#0-b*%>0C%Gg;@uC}i||E=EN|NAj%bPWTBfxnCaF|?W6T!+)AZv#>PIoGyO_fWa8UQ0;{K_k;~ qNSTg9w*N3h--0S*vRby3#0bjo{}5oF|Is(B_$L}hnw$CvN{_8 literal 0 HcmV?d00001 diff --git a/metrics/.DS_Store b/metrics/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..66f0e1e0c9e0afe28f7efe46f9eee66d28877c95 GIT binary patch literal 6148 zcmeHK&2G~`5T0!fb&L?HRH;4s!mX+jXazY`G9es#KteKt10dIqt;Uk$wPL5C4MBbl zxbgrz1Sg(=D{lhd{sc8jfv5+BU?-aW#XIfHU`AIyFn3Ul+c(iyuUocjA#oFpnKn>z#`g2 zD^yG139y0+^)f0|51Frlp;OVp!Ww>C zA)1s$l*tO8qasVos@3|!*4C|S*Eek2ah#jZD}TTzeic^ZvKx+Gh^r@IHHftRl=JL| z9Y)X6WD+#D9`d{j(>xi;!9Ccm{n@NxZ{Kbo9vu(Q&Zlo?Z{G=i(M!=xrQFguhxafxCH21~FFR@beLgJa zX{%)od9SAlvp;;jvu!>8B>wsp?le? zO)znW0mHz5!+_`yE>xhWu~sNs2L^cr09tTMfh|57M2@S`(^xCS41}pvph{)>h{04k z{H~7kG}a1LIx&6tVEWEXpHP^*JLar>J|3^PR|1Tz)E5m?c z;GbfESv|kk#Utsxb?M>6Uh6`Sp+ZDltx%?5(A%*z#8$imm4Z1}7(h>Btq>lF`4Esa Ln87gcM;Z7In^?JC literal 0 HcmV?d00001 diff --git a/metrics/parse_check.py b/metrics/parse_check.py new file mode 100644 index 0000000..7f5e750 --- /dev/null +++ b/metrics/parse_check.py @@ -0,0 +1,53 @@ +from tree_sitter import Language, Parser + +def load_tree_sitter_languages(): + """Loads language Grammars to evaluate""" + py_parser = Parser() + py_parser.set_language(Language('./tree_sitter_utils/build/my-languages.so', 'python')) + js_parser = Parser() + js_parser.set_language(Language('./tree_sitter_utils/build/my-languages.so', 'javascript')) + cpp_parser = Parser() + cpp_parser.set_language(Language('./tree_sitter_utils/build/my-languages.so', 'cpp')) + go_parser = Parser() + go_parser.set_language(Language('./tree_sitter_utils/build/my-languages.so', 'go')) + java_parser = Parser() + java_parser.set_language(Language('./tree_sitter_utils/build/my-languages.so', 'java')) + return { + "py" : py_parser, + "js" : js_parser, + "cpp" : cpp_parser, + "go" : go_parser, + "java": java_parser + } + +class check_parse: + def __init__(self): + self.language_dict = load_tree_sitter_languages() + def __call__(self,batch,lang): + """ + args: + batch : list[str] of code generated by the model + lang : lang should be one of the above language_dict keys + + returns: + dict( + parse_score = averaged out score on how many datapoints are parsed + index_parse = check if corresponding index is parsed + ) + """ + cumulative_parse_score = 0 + index_parse_list = [] + parser = self.language_dict[lang] + for inp in batch: + parsed = parser.parse(bytes(inp,"utf-8")) + inp_ind_score = int("ERROR" not in parsed.root_node.sexp()) + cumulative_parse_score+=inp_ind_score + index_parse_list.append(inp_ind_score) + return {"parse_score":cumulative_parse_score,"index_parse":index_parse_list} +if __name__ == "__main__": + Parse = check_parse() + score = Parse([""" +def a(): + if bar: + baz()"""],"py") + print(score) \ No newline at end of file diff --git a/metrics/tree_sitter_utils/.DS_Store b/metrics/tree_sitter_utils/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..643fb357ad26f079a112d68f43f45cd75b84156f GIT binary patch literal 6148 zcmeHK%}T>S5Z>*NCKa&_8`Ps54{NyJ*c#aDK-$Z(k2$ImAr<&kx$^` z=*;f6RH{eC$_&hYli8U~_S3i$(`*+@qfxb9D_h%p^>Nu6)pvHmuGGe3)7aRo?zcKO_Yb4T z@zb-M1mmz-rL1Y3zzY~NQ@nHfQ7EEo@J=!&k%YtmF+dC~Gz0ojGgcOQ7WDDN05R|r z4B+`-fg;)(Q-$*AfJT-8fH`nW0ULiDF~`zqYfKeF4+vMOfGU;i5`(LB@LM|0)|e_( z>5S{*gX=eQU7>LKb{OB1;f&h~sU-%8fzJ$J?FUlC`~T?s`d Date: Sat, 10 Jul 2021 23:41:07 +0530 Subject: [PATCH 06/15] adding parse score to evaluator --- metrics/extrinsic_eval.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/metrics/extrinsic_eval.py b/metrics/extrinsic_eval.py index 24ee15a..9d456fc 100644 --- a/metrics/extrinsic_eval.py +++ b/metrics/extrinsic_eval.py @@ -1,6 +1,9 @@ from metrics.bleu import compute_bleu +from metrics.parse_check import check_parse -def compute_metrics(references,generated) -> dict: +Parser = check_parse() #Initializing parser + +def compute_metrics(references,generated,lang) -> dict: """ Calculates various metrics and returns the calculated dict of these matrics. args: @@ -8,11 +11,12 @@ def compute_metrics(references,generated) -> dict: reference should be tokenized into a list of tokens. translation: list of translations to score. Each translation should be tokenized into a list of tokens. + lang(str) : The language generated code belongs to returns: A dicitonary with different metrics intact. """ metrics_dict = {} #Update as in new metrics are added over here. metrics_dict["smoothed_bleu_4"] = compute_bleu(references,generated,smooth=True) metrics_dict["bleu_4"] = compute_bleu(references,generated,smooth=False) - + metrics_dict["parse_score"] = Parser(generated,lang)["parse_score"] return metrics_dict \ No newline at end of file From 36bf7d245e35d736c431ac3b374763dc871e3dac Mon Sep 17 00:00:00 2001 From: ncoop57 Date: Sat, 10 Jul 2021 23:31:55 +0000 Subject: [PATCH 07/15] Add additional repos and submodules --- .gitmodules | 6 ++++++ dependency_repos/apps | 1 + dependency_repos/human-eval | 1 + 3 files changed, 8 insertions(+) create mode 160000 dependency_repos/apps create mode 160000 dependency_repos/human-eval diff --git a/.gitmodules b/.gitmodules index c0bb54c..22f1e92 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,9 @@ [submodule "dependency_repos/github-downloader"] path = dependency_repos/github-downloader url = https://github.com/EleutherAI/github-downloader +[submodule "dependency_repos/apps"] + path = dependency_repos/apps + url = https://github.com/hendrycks/apps.git +[submodule "dependency_repos/human-eval"] + path = dependency_repos/human-eval + url = https://github.com/openai/human-eval diff --git a/dependency_repos/apps b/dependency_repos/apps new file mode 160000 index 0000000..f834ca7 --- /dev/null +++ b/dependency_repos/apps @@ -0,0 +1 @@ +Subproject commit f834ca7d7405935376aabb5830edd0c42635824e diff --git a/dependency_repos/human-eval b/dependency_repos/human-eval new file mode 160000 index 0000000..463c980 --- /dev/null +++ b/dependency_repos/human-eval @@ -0,0 +1 @@ +Subproject commit 463c980b59e818ace59f6f9803cd92c749ceae61 From 22fcb3c5de371caf0f2dccc0d66b45be907f7ea6 Mon Sep 17 00:00:00 2001 From: ncoop57 Date: Sun, 11 Jul 2021 02:24:38 +0000 Subject: [PATCH 08/15] Add initial evaluation code --- .../apps_utils/generate_gpt_codes.py | 102 ++++ scripts/evaluation/apps_utils/reindent.py | 227 ++++++++ .../apps_utils/test_one_solution.py | 158 +++++ scripts/evaluation/apps_utils/testing_util.py | 544 ++++++++++++++++++ scripts/evaluation/code_search_net.py | 4 + scripts/evaluation/concode.py | 23 + scripts/evaluation/evaluate.py | 165 ++++++ scripts/evaluation/human_eval.jsonl | 164 ++++++ .../evaluation/human_eval.jsonl_results.jsonl | 164 ++++++ scripts/evaluation/human_eval_bench.py | 0 scripts/evaluation/metrics/bleu.py | 133 +++++ scripts/evaluation/metrics/extrinsic_eval.py | 46 ++ 12 files changed, 1730 insertions(+) create mode 100644 scripts/evaluation/apps_utils/generate_gpt_codes.py create mode 100644 scripts/evaluation/apps_utils/reindent.py create mode 100644 scripts/evaluation/apps_utils/test_one_solution.py create mode 100644 scripts/evaluation/apps_utils/testing_util.py create mode 100644 scripts/evaluation/code_search_net.py create mode 100644 scripts/evaluation/concode.py create mode 100644 scripts/evaluation/evaluate.py create mode 100644 scripts/evaluation/human_eval.jsonl create mode 100644 scripts/evaluation/human_eval.jsonl_results.jsonl create mode 100644 scripts/evaluation/human_eval_bench.py create mode 100644 scripts/evaluation/metrics/bleu.py create mode 100644 scripts/evaluation/metrics/extrinsic_eval.py diff --git a/scripts/evaluation/apps_utils/generate_gpt_codes.py b/scripts/evaluation/apps_utils/generate_gpt_codes.py new file mode 100644 index 0000000..f26ca22 --- /dev/null +++ b/scripts/evaluation/apps_utils/generate_gpt_codes.py @@ -0,0 +1,102 @@ +# MIT License + +# Copyright (c) 2021 Dan Hendrycks and contributors. + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +""" +Run a tranined model to generate Python code. +""" + +import io +import json +import logging +import math +import random +import numpy as np +import os +import pprint +import sys +import time +import transformers +import torch + +from apps_utils.reindent import run as run_reindent + +# for timing and debugging +from datetime import datetime, date +from tqdm import tqdm + + +def reindent_code(codestr): + """ + Given code string, reindent it in the same way that the + Github dataset was indented + """ + codestr = io.StringIO(codestr) + ret = io.StringIO() + + run_reindent( + codestr, + ret, + config={ + "dry-run": False, + "help": False, + "to": 10, + "from": -1, + "tabs": True, + "encoding": "utf-8", + "is-tabs": False, + "tabsize": 10, + "all-tabs": False, + }, + ) + + return ret.getvalue() + + +def generate_prompt( + test_case_path, prompt_path, solutions_path, tokenizer, starter_path=None +): + _input = "\nQUESTION:\n" + with open(prompt_path, "r") as f: + data = f.readlines() + data = "".join(data) + _input += data + if starter_path != None: + with open(starter_path, "r") as f: + data = f.readlines() + data = "".join(data) + data = "\n" + data # + "\n" + _input += data + else: + # _input += "\n\n" + pass + + with open(test_case_path, "r") as f: + data = json.load(f) + if not data.get("fn_name"): + _input += "\nUse Standard Input format" # \n" + else: + _input += "\nUse Call-Based format" # \n" + + _input += "\nANSWER:\n" + + return _input diff --git a/scripts/evaluation/apps_utils/reindent.py b/scripts/evaluation/apps_utils/reindent.py new file mode 100644 index 0000000..78f7141 --- /dev/null +++ b/scripts/evaluation/apps_utils/reindent.py @@ -0,0 +1,227 @@ +# MIT License + +# Copyright (c) 2021 Dan Hendrycks and contributors. + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +""" +Reindent files. +""" + +from __future__ import print_function +import sys +import getopt +import codecs +import tempfile +import shutil +import os + + +def _find_indentation(line, config): + if len(line) and line[0] in (" ", "\t") and not line.isspace(): + if line[0] == "\t": + config['is-tabs'] = True + # Find indentation + i = 0 + for char in list(line): + if char not in (" ", "\t"): + break + i += 1 + config["from"] = i + + +def find_indentation(line, config): + # Find indentation level used in file + if config['from'] < 0: + _find_indentation(line, config) + + if config['from'] >= 0: + # Set old indent + indent = " " if not config['is-tabs'] else "\t" + indent = indent * config['from'] + + # Set new indent + newindent = " " if not config['tabs'] else "\t" + if not config['tabs']: + newindent = newindent * config['to'] + + return indent, newindent + + # Continue to the next line, indentation not found + return False + + +def replace_inline_tabs(content, config): + newcontent = "" + imagined_i = 0 + for i in range(0, len(content)): + char = content[i] + if char == '\t': + spaces = config['tabsize']-(imagined_i % config['tabsize']) + newcontent += " " * spaces + imagined_i += spaces + else: + newcontent += char + imagined_i += 1 + return newcontent + + +def run(fd_in, fd_out, config): + from reindent_4_spaces import Reindenter + import io + + inter = io.StringIO() + ri = Reindenter(fd_in) + ri.run() + ri.write(inter) + fd_in = inter + fd_in.seek(0) + + while True: + line = fd_in.readline() + if not line: + break + line = line.rstrip('\r\n') + + # Find indentation style used in file if not set + if config['from'] < 0: + indent = find_indentation(line, config) + if not indent: + print(line, file=fd_out) + continue + indent, newindent = indent + + # Find current indentation level + level = 0 + while True: + whitespace = line[:len(indent) * (level + 1)] + if whitespace == indent * (level + 1): + level += 1 + else: + break + + content = line[len(indent) * level:] + if config['all-tabs']: + content = replace_inline_tabs(content, config) + + line = (newindent * level) + content + print(line, file=fd_out) + # print(config) + + +def run_files(filenames, config): + for filename in filenames: + with codecs.open(filename, encoding=config['encoding']) as fd_in: + if config['dry-run']: + print("Filename: %s" % filename) + fd_out = sys.stdout + else: + fd_out = tempfile.NamedTemporaryFile(mode='wb', delete=False) + fd_out.close() + fd_out = codecs.open(fd_out.name, "wb", encoding=config['encoding']) + + run(fd_in, fd_out, config) + + if not config["dry-run"]: + fd_out.close() + shutil.copy(fd_out.name, filename) + os.remove(fd_out.name) + + +def main(args): + config = { + "dry-run": False, + "help": False, + "to": 4, + "from": -1, + "tabs": False, + "encoding": "utf-8", + "is-tabs": False, + "tabsize": 4, + "all-tabs": False + } + possible_args = { + "d": "dry-run", + "h": "help", + "t:": "to=", + "f:": "from=", + "n": "tabs", + "e:": "encoding=", + "s:": "tabsize=", + "a": "all-tabs", + } + optlist, filenames = getopt.getopt( + args[1:], + "".join(possible_args.keys()), + possible_args.values() + ) + + shortargs, longargs = [], [] + for shortarg in possible_args: + shortargs.append(shortarg.rstrip(":")) + longargs.append(possible_args[shortarg].rstrip("=")) + + for opt, val in optlist: + opt = opt.lstrip("-") + if opt in shortargs: + opt = longargs[shortargs.index(opt)] + if isinstance(config[opt], bool): + config[opt] = True + elif isinstance(config[opt], int): + config[opt] = int(val) + else: + config[opt] = val + + if config['help']: + help = """ + Usage: %s [options] filename(s) + Options: + -h, --help Show this message + -d, --dry-run Don't save anything, just print + the result + -t , --to Convert to this number of spaces + (default: 4) + -f , --from Convert from this number of spaces + (default: auto-detect, will also + detect tabs) + -n, --tabs Don't convert indentation to spaces, + convert to tabs instead. -t and + --to will have no effect. + -a, --all-tabs Also convert tabs used for alignment + in the code (Warning: will replace + all tabs in the file, even if inside + a string) + -s , --tabsize Set how many spaces one tab is + (only has an effect on -a, default: 4) + -e , --encoding Open files with specified encoding + (default: utf-8) + """ % args[0] + + # Also removes 8 leading spaces to remove our indentation + print("\n".join([x[8:] for x in help[1:].split("\n")])) + sys.exit(0) + + if filenames: + run_files(filenames, config) + else: + run(sys.stdin, sys.stdout, config) + +if __name__ == "__main__": + main(sys.argv) diff --git a/scripts/evaluation/apps_utils/test_one_solution.py b/scripts/evaluation/apps_utils/test_one_solution.py new file mode 100644 index 0000000..fafc307 --- /dev/null +++ b/scripts/evaluation/apps_utils/test_one_solution.py @@ -0,0 +1,158 @@ +# MIT License + +# Copyright (c) 2021 Dan Hendrycks and contributors. + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +Run solutions from one problem. +""" + +import io +import json +import logging +import math +import numpy as np +import os +import pprint +import sys +import apps_utils.testing_util as test_util +import time + +# for timing debugging +from datetime import datetime, date +from pathlib import Path +from tqdm import tqdm + +from typing import List + + +def print_results(results, args): + res = [] + per_prob_res = [] + all_correct = [] + for index in results: + res.extend(results[index]) + per_prob_res.append(np.mean(results[index])) + all_correct.append(np.all(results[index])) + tmp_results = res + compile_errors = len(tmp_results[tmp_results == -2]) + runtime_errors = len(tmp_results[tmp_results == -1]) + failures = len(tmp_results[tmp_results == False]) + successes = len(tmp_results[tmp_results == True]) + total_testcases = len(res) + if args.debug: + print( + f"number of compile errors = {compile_errors} avg = {compile_errors / total_testcases }" + ) + print( + f"number of runtime errors = {runtime_errors} avg = {runtime_errors / total_testcases}" + ) + print(f"number of test cases run = {total_testcases}") + + print( + f"Test Case Average (average accuracy over problems) = {np.mean(per_prob_res)}" + ) + print( + f"Strict Accuracy (all test cases passed / total problems) = {np.mean(all_correct)}" + ) + + +def eval_and_save_problems(test_loc, save): + test_path = Path(test_loc) + problems = list(test_path.glob("*/")) + + print(len(problems)) + gpt_codes = {} + gpt_bleu = {} + gpt_codebleu = {} + results = {} + codes_loc = os.path.join(save, f"all_codes.json") + # if not os.path.exists(codes_loc): + # codes_loc = os.path.join(args.save, f"{args.start}-{args.end}_codes.json") + + if os.path.exists(codes_loc): + results_loc = os.path.join(save, f"all_results.json") + print(codes_loc, results_loc) + + with open(codes_loc, "r") as f: + gpt_codes = json.load(f) + + # main eval loop + for index, problem in enumerate(tqdm(problems[:2])): + try: + # if args.debug: + # print(f"\n\nproblem path = {problem}") + output_str = gpt_codes[str(index)] + except: + print("CANNOT FIND OUTPUT_STR FOR", problem) + continue + prob_path = problem # os.path.join(args.root, problem) + + # with open(os.path.join(prob_path, "solutions.json"), "r") as f: + # sols = json.load(f) + + if not os.path.exists(save): + os.makedirs(save) + + res = [] + # for o_idx, o in enumerate(output_str): + # print(o) + # if args.debug: + # print(f"\nTesting solution {o_idx}") + curr_res = [-2] + try: + curr_res = test_util.run_test( + prob_path=prob_path, test=output_str, debug=False # args.debug + ) + fixed = [] + for e in curr_res: + if isinstance(e, np.ndarray): + e = e.item(0) + if isinstance(e, np.bool_): + e = bool(e) + fixed.append(e) + curr_res = fixed + if not np.all(curr_res): + print(f"Results were not all True: {curr_res}") + except Exception as e: + print(f"test framework exception = {repr(e)}{e}\n") + break + finally: + assert isinstance(curr_res, list) + res.append(curr_res) + + # if args.debug: + # print( + # f"\nHow to read results [-2] = compile error, [-1] = runtime error [False] = failed test case [True] = passed test case" + # ) + # print(f"results = {res}") + + results[index] = res + + with open(results_loc, "w") as f: + try: + f.write(json.dumps(results)) + except Exception as e: + import pdb + + pdb.set_trace() + print("didn't save problem due to {e}") + + return results diff --git a/scripts/evaluation/apps_utils/testing_util.py b/scripts/evaluation/apps_utils/testing_util.py new file mode 100644 index 0000000..2c5444f --- /dev/null +++ b/scripts/evaluation/apps_utils/testing_util.py @@ -0,0 +1,544 @@ +import argparse +import json +import os +import sys +import io +import faulthandler + +# used for debugging to time steps +from datetime import datetime + +# to run the solution files we're using a timing based approach +import signal + +import numpy as np +# for capturing the stdout +from io import StringIO +from typing import get_type_hints +from typing import List, Tuple +# used for testing the code that reads from input +from unittest.mock import patch, mock_open + +from pyext import RuntimeModule + +from enum import Enum +class CODE_TYPE(Enum): + call_based = 0 + standard_input = 1 + +# stuff for setting up signal timer +class TimeoutException(Exception): + pass +def timeout_handler(signum, frame): + print("alarm went off") + #return + raise TimeoutException +signal.signal(signal.SIGALRM, timeout_handler) +timeout = 4 # seconds + +# used to capture stdout as a list +# from https://stackoverflow.com/a/16571630/6416660 +# alternative use redirect_stdout() from contextlib +class Capturing(list): + def __enter__(self): + self._stdout = sys.stdout + sys.stdout = self._stringio = StringIO() + # Make closing the StringIO a no-op + self._stringio.close = lambda x: 1 + return self + def __exit__(self, *args): + self.extend(self._stringio.getvalue().splitlines()) + del self._stringio # free up some memory + sys.stdout = self._stdout + + +def parse_args(): + parser = argparse.ArgumentParser(description="Utility for testing code generation.") + parser.add_argument("-v", "--verbosity-level", action="store", type=int, + help="") + parser.add_argument("-s", "--source", type=str, default="leetcode", + choices=["leetcode", "atcoder", "codewars",], + help="which data source to gather from.") + parser.add_argument("-d", "--data", type=str, default="question", + choices=["question", "q", "solutions", "sol", "s", "starter", "tests", "t"], + help="which type of data to receive.") + parser.add_argument("-n", "--number", type=int, default=0, + help="which problem to query.") + + args = parser.parse_args() + return args + + +def get_valid_problems(data_dir="leetcode"): + # these are unnecessary atm + if data_dir == "leetcode": + root = os.path.join(args.source, "data") + elif data_dir == "atcoder": + pass + + root = os.path.join(data_dir, "data") + if os.path.exists(os.path.join(data_dir, "valid_problems.json")): + with open(os.path.join(data_dir, "valid_problems.json"), "r") as f: + return json.load(f) + + # after we compute it once let's save it and load that instead + # TODO determine if might be better to reload each time + tmp = os.listdir(root) + valid_probs = [] + for folder in tmp: + prob_path = os.path.join(root, folder) + files = os.listdir(prob_path) + #TODO add more validity checks + if "input_output.json" in files or "sols.json" in files: + valid_probs.append(prob_path) + valid_probs = sorted(valid_probs) + #with open(os.path.join(args.source,"valid_problems.json"), "w") as f: + # json.dump(valid_probs, f) + return valid_probs + + +def get_question(problem_list, prob_index): + root = problem_list[prob_index] + #print("get q", root) + if os.path.exists(os.path.join(root, "question.txt")): + with open(os.path.join(root, "question.txt")) as f: + question = f.readlines() + else: + print("question prompt not found") + question = "" + question = "".join(question) + return question + + +def get_solutions(problem_list, prob_index): + root = problem_list[prob_index] + if os.path.exists(os.path.join(root, "solutions.json")): + with open(os.path.join(root, "solutions.json")) as f: + sols = json.load(f) + return sols + + +def run_test(prob_path:str=None, problem_list:List[str]=None, prob_index:int=None, + test:str=None, debug:bool=False): + """ + if test is not None it'll try to run the code. + otherwise it'll just return an input and output pair. + """ + if prob_path is None and problem_list is None: + print("please provide either prob_path or problem_list") + exit() + + if debug: + print(f"start = {datetime.now().time()}") + if prob_path is not None: + root = prob_path + elif problem_list is not None: + root = problem_list[prob_index] + + if os.path.exists(os.path.join(root, "input_output.json")): + with open(os.path.join(root, "input_output.json")) as f: + in_outs = json.load(f) + if debug: + print(f"test cases json = {in_outs['inputs']} {in_outs['outputs']}") + + if in_outs.get("fn_name") is None: + which_type = CODE_TYPE.standard_input # Standard input + method_name = None + else: + which_type = CODE_TYPE.call_based # Call-based + method_name = in_outs["fn_name"] + if debug: + print(f"loaded json = {datetime.now().time()}") + + #else: + # continue + if test is None: + return in_outs + elif test is not None: + results = [] + sol = "import sys\nimport time\nimport itertools\nfrom itertools import accumulate, product, permutations, combinations\nimport collections\nfrom collections import Counter, OrderedDict, deque, defaultdict, ChainMap\nfrom functools import lru_cache\nimport math\nfrom math import sqrt, sin, cos, tan, ceil, fabs, floor, gcd, exp, log, log2\nimport fractions\nfrom typing import List, Tuple\nimport numpy as np\nimport random\nimport heapq\nfrom heapq import *\n" + if debug: + print(f"loading test code = {datetime.now().time()}") + + if which_type == CODE_TYPE.call_based: + sol += test + if debug: # or True: + print(f"sol = {sol}") + signal.alarm(timeout) + try: + tmp_sol = RuntimeModule.from_string("tmp_sol", "", sol) + if "class Solution" not in test: + tmp = tmp_sol + else: + tmp = tmp_sol.Solution() + signal.alarm(0) + except Exception as e: + signal.alarm(0) + print(f"type 0 compilation error = {e}") + results.append(-2) + return results + signal.alarm(0) + + elif which_type == CODE_TYPE.standard_input: + # sol + tmp_test = test.split("\n") + + new_test = [] + for x in tmp_test: + if (not x.startswith("from ")) and (not x.startswith("import ")): + new_test.append("\t" + x + "\n") + else: + new_test.append(x + "\n") + tmp_test = new_test + + new_test = "" + started = False + for i in tmp_test: + if i.startswith("\t") and not started: + new_test += "stdin = sys.stdin\nstdout = sys.stdout\n" + new_test += "def code():\n" + new_test += i + started = True + elif started and ((i.startswith("from ")) or (i.startswith("import "))): + new_test += "\t" + i + else: + new_test += i + tmp_test = new_test + + sol += tmp_test + if debug: + print(f"sol = {sol}") + # print(f"{o}") + method_name = "code" + signal.alarm(timeout) + try: + tmp_sol = RuntimeModule.from_string("tmp_sol", "", sol) + tmp = tmp_sol + signal.alarm(0) + except Exception as e: + signal.alarm(0) + print(f"type 1 compilation error = {e}") + results.append(-2) + return results + signal.alarm(0) + if debug: + print(f"get method = {datetime.now().time()}") + + try: + method = getattr(tmp, method_name) # get_attr second arg must be str + except: + signal.alarm(0) + e = sys.exc_info() + print(f"unable to get function error = {e}") + return results + + for index, inputs in enumerate(in_outs["inputs"]): + # JSON forces dictionaries to have string keys; this undoes this (assuming a singleton list) + try: + if isinstance(inputs[0], dict): + inputs = [{int(k): v for k,v in inputs[0].items()}] + except: + True + try: + if isinstance(in_outs["outputs"][index], dict): + in_outs["outputs"][index] = [{int(k): v for k,v in in_outs["outputs"][index].items()}] + except: + True + try: + if isinstance(in_outs["outputs"][index][0], dict): + in_outs["outputs"][index] = [{int(k): v for k,v in in_outs["outputs"][index][0].items()}] + except: + True + + if debug: + print(f"time: {datetime.now().time()} testing index = {index} inputs = {inputs}, {type(inputs)}. type = {which_type}") + if which_type == CODE_TYPE.call_based: # Call-based + signal.alarm(timeout) + faulthandler.enable() + try: + # print("------------") + # print(inputs) + output = method(*inputs) + + # ground truth sequences are not tuples + if isinstance(output, tuple): + output = list(output) + + tmp_result = output == in_outs["outputs"][index] + if isinstance(in_outs["outputs"][index], list) and in_outs["outputs"][index]: + tmp_result = tmp_result or (output == in_outs["outputs"][index][0]) + + # ground truth sequences are not tuples + try: + if isinstance(output[0], tuple): + tmp_result = tmp_result or ([list(x) for x in output] == in_outs["outputs"][index][0]) + except: + True + results.append(tmp_result) + + # reset the alarm + signal.alarm(0) + except Exception as e: + signal.alarm(0) + faulthandler.disable() + print(f"Standard input runtime error or time limit exceeded error = {e}") + results.append(-1) + continue + faulthandler.disable() + signal.alarm(0) + if debug: + print(f"outputs = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}") + elif which_type == CODE_TYPE.standard_input: # Standard input + faulthandler.enable() + signal.alarm(timeout) + passed = False + + if isinstance(inputs, list): + inputs = "\n".join(inputs) + if isinstance(in_outs['outputs'][index], list): + in_outs['outputs'][index] = "\n".join(in_outs['outputs'][index]) + + with Capturing() as output: + try: + call_method(method, inputs) + # reset the alarm + signal.alarm(0) + passed = True + except Exception as e: + # runtime error or took too long + signal.alarm(0) + print(f"Call-based runtime error or time limit exceeded error = {repr(e)}{e}") + results.append(-1) + signal.alarm(0) + + if not passed: + if debug: + nl = "\n" + if not isinstance(inputs, list): + print(f"not passed output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]}") + else: + print(f"not passed output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}") + continue + + if passed and debug: + print(f"==> output = {output}, test outputs = {in_outs['outputs'][index]}") + + if custom_compare_(output, in_outs['outputs'][index]): + tmp_result = True + results.append(tmp_result) + continue + + # ground truth sequences are expressed as lists not tuples + if isinstance(output, tuple): + output = list(output) + + tmp_result = False + try: + tmp_result = (output == [in_outs["outputs"][index]]) + if isinstance(in_outs["outputs"][index], list): + tmp_result = tmp_result or (output == in_outs["outputs"][index]) + if isinstance(output[0], str): + tmp_result = tmp_result or ([e.strip() for e in output] == in_outs["outputs"][index]) + except Exception as e: + print(f"Failed check1 exception = {e}") + pass + + if tmp_result == True: + results.append(tmp_result) + continue + + # try one more time without \n + if isinstance(in_outs["outputs"][index], list): + for tmp_index, i in enumerate(in_outs["outputs"][index]): + in_outs["outputs"][index][tmp_index] = i.split("\n") + in_outs["outputs"][index][tmp_index] = [x.strip() for x in in_outs["outputs"][index][tmp_index] if x] + else: + in_outs["outputs"][index] = in_outs["outputs"][index].split("\n") + in_outs["outputs"][index] = list(filter(len, in_outs["outputs"][index])) + in_outs["outputs"][index] = list(map(lambda x:x.strip(), in_outs["outputs"][index])) + + try: + tmp_result = (output == [in_outs["outputs"][index]]) + if isinstance(in_outs["outputs"][index], list): + tmp_result = tmp_result or (output == in_outs["outputs"][index]) + except Exception as e: + print(f"Failed check2 exception = {e}") + pass + + if tmp_result == True: + results.append(tmp_result) + continue + + # try by converting the output into a split up list too + if isinstance(output, list): + output = list(filter(len, output)) + + if debug: + nl = "\n" + if not isinstance(inputs, list): + print(f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]}") + else: + print(f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}") + + if tmp_result == True: + results.append(tmp_result) + continue + + try: + tmp_result = (output == [in_outs["outputs"][index]]) + if isinstance(in_outs["outputs"][index], list): + tmp_result = tmp_result or (output == in_outs["outputs"][index]) + except Exception as e: + print(f"Failed check3 exception = {e}") + pass + + try: + output_float = [float(e) for e in output] + gt_float = [float(e) for e in in_outs['outputs'][index]] + tmp_result = tmp_result or ((len(output_float) == len(gt_float)) and np.allclose(output_float, gt_float)) + except Exception as e: + pass + try: + if isinstance(output[0], list): + output_float = [float(e) for e in output[0]] + gt_float = [float(e) for e in in_outs['outputs'][index][0]] + tmp_result = tmp_result or ((len(output_float) == len(gt_float)) and np.allclose(output_float, gt_float)) + except Exception as e: + pass + + if tmp_result == True: + results.append(tmp_result) + continue + + # try by converting the stuff into split up list + if isinstance(in_outs["outputs"][index], list): + for tmp_index, i in enumerate(in_outs["outputs"][index]): + in_outs["outputs"][index][tmp_index] = set(i.split()) + else: + in_outs["outputs"][index] = set(in_outs["outputs"][index].split()) + + try: + tmp_result = (output == in_outs["outputs"][index]) + except Exception as e: + print(f"Failed check4 exception = {e}") + continue + + if tmp_result == True: + results.append(tmp_result) + continue + + # try by converting the output into a split up list too + if isinstance(output, list): + for tmp_index, i in enumerate(output): + output[tmp_index] = i.split() + output = list(filter(len, output)) + for tmp_index, i in enumerate(output): + output[tmp_index] = set(i) + else: + output = output.split() + output = list(filter(len, output)) + output = set(output) + + try: + tmp_result = (set(frozenset(s) for s in output) == set(frozenset(s) for s in in_outs["outputs"][index])) + except Exception as e: + print(f"Failed check5 exception = {e}") + + + # if they are all numbers, round so that similar numbers are treated as identical + try: + tmp_result = tmp_result or (set(frozenset(round(float(t),3) for t in s) for s in output) ==\ + set(frozenset(round(float(t),3) for t in s) for s in in_outs["outputs"][index])) + except Exception as e: + print(f"Failed check6 exception = {e}") + + if tmp_result == True and debug: + print("PASSED") + + results.append(tmp_result) + + if debug: + nl = "\n" + if not isinstance(inputs, list): + print(f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs.replace(nl,' new-line ')}, {type(inputs)}, {output == [in_outs['outputs'][index]]}") + else: + print(f"output = {output}, test outputs = {in_outs['outputs'][index]}, inputs = {inputs}, {type(inputs)}, {output == [in_outs['outputs'][index]]}") + + + return results + +def custom_compare_(output, ground_truth): + + if isinstance(output, list): + output_1 = "\n".join(output) + if stripped_string_compare(output_1, ground_truth): + return True + + if isinstance(output, list): + output_2 = [o.lstrip().rstrip() for o in output] + output_2 = "\n".join(output_2) + if stripped_string_compare(output_2, ground_truth): + return True + + return False + +def stripped_string_compare(s1, s2): + s1 = s1.lstrip().rstrip() + s2 = s2.lstrip().rstrip() + return s1 == s2 + +def call_method(method, inputs): + + if isinstance(inputs, list): + inputs = "\n".join(inputs) + + inputs_line_iterator = iter(inputs.split("\n")) + + # sys.setrecursionlimit(10000) + + # @patch('builtins.input', side_effect=inputs.split("\n")) + @patch('builtins.open', mock_open(read_data=inputs)) + @patch('sys.stdin', StringIO(inputs)) + @patch('sys.stdin.readline', lambda *args: next(inputs_line_iterator)) + @patch('sys.stdin.readlines', lambda *args: inputs.split("\n")) + @patch('sys.stdin.read', lambda *args: inputs) + # @patch('sys.stdout.write', print) + def _inner_call_method(_method): + try: + return _method() + except SystemExit as e: + pass + finally: + pass + return _inner_call_method(method) + +def main(args): + print(args) + problem_list = sorted(get_valid_problems(args.source)) + print(f"number of problems = {len(problem_list)}") + prob_index = args.number + print(f"problem is {problem_list[prob_index]}") + + # This checks it correctly loaded. remove this later + assert prob_index < len(problem_list) + + if args.data == "q" or args.data == "question": + tmp = get_question(problem_list, prob_index) + print("q", tmp) + elif args.data in ["solutions", "sol", "s",]: + tmp = get_solutions(problem_list, prob_index) + print("sol", tmp) + elif args.data == "starter": + tmp = get_starter(problem_list, prob_index) + print("starter", tmp) + elif args.data in ["test", "t"]: + # test it with sols + sols = get_solutions(problem_list, prob_index) + tmp = run_test(problem_list, prob_index, test=sols[0]) + + print("results = ", tmp) + print("-2 = compile error, -1 is runtime error, False failed test, True passed test") + +if __name__ == "__main__": + args = parse_args() + main(args) diff --git a/scripts/evaluation/code_search_net.py b/scripts/evaluation/code_search_net.py new file mode 100644 index 0000000..49ac5ad --- /dev/null +++ b/scripts/evaluation/code_search_net.py @@ -0,0 +1,4 @@ +from datasets import load_dataset + +dataset = load_dataset("code_x_glue_ct_code_to_text", "go") +print(dataset) diff --git a/scripts/evaluation/concode.py b/scripts/evaluation/concode.py new file mode 100644 index 0000000..dd90f34 --- /dev/null +++ b/scripts/evaluation/concode.py @@ -0,0 +1,23 @@ +import pandas as pd + +from datasets import load_dataset, load_metric +from fastcore.script import * +from pathlib import Path + +bleu = load_metric("sacrebleu") + +predictions = ["hello there kenobi", "foo bar foobar"] +references = [ + ["hello there general kenobi"], + ["foo bar foobar"], # , "hello there !"], # , "foo bar foobar"], +] + + +@call_parse +def main(concode_path: Param("Path to the concode data in CodeXGLUE", str)): + concode_path = Path(concode_path) + dataset = load_dataset("json", data_files=str(concode_path / "test.json")) + print(dataset) + results = bleu.compute(predictions=predictions, references=references) + print(list(results.keys())) + print(round(results["score"], 1)) diff --git a/scripts/evaluation/evaluate.py b/scripts/evaluation/evaluate.py new file mode 100644 index 0000000..4fbd421 --- /dev/null +++ b/scripts/evaluation/evaluate.py @@ -0,0 +1,165 @@ +import json +import torch +import pandas as pd + +# import apps.eval.reident + +from apps_utils.generate_gpt_codes import generate_prompt +from apps_utils.test_one_solution import eval_and_save_problems +from datasets import load_dataset, load_metric +from fastcore.script import * +from human_eval.data import write_jsonl, read_problems +from pathlib import Path +from metrics.extrinsic_eval import compute_metrics +from subprocess import check_output +from transformers import AutoTokenizer, AutoModelWithLMHead + +bleu = load_metric("sacrebleu") +tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neo-125M") +model = AutoModelWithLMHead.from_pretrained( + "/home/nathan/gpt-code-clippy/data/APPS/models/1.5B" +) + + +def generate_text(prompt): + # print(prompt) + input_ids = torch.LongTensor(tokenizer.encode(prompt, verbose=False)).unsqueeze( + 0 + ) # .cuda() + output_ids = model.generate( + input_ids, + num_beams=2, + early_stopping=True, + max_length=1024 - len(input_ids), + ) + output_str = tokenizer.decode(output_ids[0]) + return output_str + # # "a", "=", "b", "\n", "y", "=", "a", "+", "1" + # return "a = b \n y = a + 1" + + +def _eval_concode(path): + # TODO: format input to model same as App and OpenAI HumanEval datasets are formatted + data = load_dataset("json", data_files=str(path / "test.json"))["train"] + predictions = [[]] + references = [] + for example in data: + output = generate_text(example["nl"]) + predictions[0].append(output.split(" ")) + references.append(example["code"].split(" ")) + results = compute_metrics(predictions, references) + print(f"Bleu score for Concode dataset: {results}") + + +def _eval_apps(path): + gpt_codes = {} + prob_paths = sorted(path.glob("*/")) + # map prob_paths to strings and save as a json file + str_paths = [str(p) for p in prob_paths] + with open(path / "test.json", "w") as f: + json.dump(str_paths, f) + for index, prob_path in enumerate(prob_paths[:2]): + test_case_path = prob_path / "input_output.json" + prompt_path = prob_path / "question.txt" + starter_path = prob_path / "starter_code.py" + solutions_path = prob_path / "solutions.json" + if not starter_path.exists(): + starter_path = None + if not test_case_path.exists() or not prompt_path.exists(): + continue + prompt = generate_prompt( + Args(), + test_case_path, + prompt_path, + solutions_path, + tokenizer, + starter_path=starter_path, + ) + output = generate_text(prompt) + print(output) + # print(output) + gpt_codes[index] = output + # print(output) + + with open(path.parent / "all_codes.json", "w") as f: + json.dump(gpt_codes, f) + + eval_and_save_problems(path, path.parent) + + # execute bash command to run eval script + # results = check_output( + # [ + # # python3 test_one_solution.py -t /path/to/apps/test --save /path/to/save_dir --print_results + # "python", + # "./apps_utils/test_one_solution.py", + # "-t", + # str(path), + # "--save", + # str(path.parent), + # "--print_results", + # ] + # ).decode("utf-8") + + +# test_case_path = os.path.join(prob_path, "input_output.json") +# prompt_path = os.path.join(prob_path, "question.txt") +# starter_path = os.path.join(prob_path, "starter_code.py") +# solutions_path = os.path.join(prob_path, "solutions.json") +# generate_prompt(args, test_case_path, prompt_path, solutions_path, tokenizer, starter_path=None) + + +def _eval_human_eval(path): + problems = read_problems() + num_samples_per_task = 1 + samples = [ + dict( + task_id=task_id, + completion=generate_text(problems[task_id]["prompt"]), + ) + for task_id in problems + for _ in range(num_samples_per_task) + ] + write_jsonl("human_eval.jsonl", samples) + # execute bash command to run eval script + results = check_output( + [ + "python", + path / "evaluate_functional_correctness.py", + "human_eval.jsonl", + ] + ).decode("utf-8") + + print(results) + + +@call_parse +def main( + concode_path: Param("Path to the concode data in CodeXGLUE", str), + apps_path: Param("Path to the the App dataset", str), + human_eval_path: Param("Path to the human eval dataset", str), +): + concode_path = Path(concode_path) + apps_path = Path(apps_path) + human_eval_path = Path(human_eval_path) + # _eval_concode(concode_path) + # _eval_human_eval(human_eval_path) + _eval_apps(apps_path) + # dataset = load_dataset("json", data_files=str(concode_path / "test.json")) + # print(dataset) + # results = bleu.compute(predictions=predictions, references=references) + # print(list(results.keys())) + # print(round(results["score"], 1)) + + +# problems = read_problems() +# print(problems) +# num_samples_per_task = 200 +# samples = [ +# dict( +# task_id=task_id, +# completion=generate_text(problems[task_id]["prompt"]), +# ) +# for task_id in problems[:1] +# for _ in range(num_samples_per_task) +# ] +# write_jsonl("human_eval.jsonl", samples) diff --git a/scripts/evaluation/human_eval.jsonl b/scripts/evaluation/human_eval.jsonl new file mode 100644 index 0000000..beafda1 --- /dev/null +++ b/scripts/evaluation/human_eval.jsonl @@ -0,0 +1,164 @@ +{"task_id": "HumanEval/0", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/1", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/2", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/3", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/4", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/5", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/6", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/7", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/8", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/9", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/10", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/11", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/12", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/13", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/14", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/15", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/16", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/17", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/18", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/19", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/20", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/21", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/22", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/23", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/24", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/25", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/26", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/27", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/28", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/29", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/30", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/31", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/32", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/33", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/34", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/35", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/36", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/37", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/38", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/39", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/40", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/41", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/42", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/43", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/44", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/45", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/46", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/47", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/48", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/49", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/50", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/51", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/52", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/53", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/54", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/55", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/56", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/57", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/58", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/59", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/60", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/61", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/62", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/63", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/64", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/65", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/66", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/67", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/68", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/69", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/70", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/71", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/72", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/73", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/74", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/75", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/76", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/77", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/78", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/79", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/80", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/81", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/82", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/83", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/84", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/85", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/86", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/87", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/88", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/89", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/90", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/91", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/92", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/93", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/94", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/95", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/96", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/97", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/98", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/99", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/100", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/101", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/102", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/103", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/104", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/105", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/106", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/107", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/108", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/109", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/110", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/111", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/112", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/113", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/114", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/115", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/116", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/117", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/118", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/119", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/120", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/121", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/122", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/123", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/124", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/125", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/126", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/127", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/128", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/129", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/130", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/131", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/132", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/133", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/134", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/135", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/136", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/137", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/138", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/139", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/140", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/141", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/142", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/143", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/144", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/145", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/146", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/147", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/148", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/149", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/150", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/151", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/152", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/153", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/154", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/155", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/156", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/157", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/158", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/159", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/160", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/161", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/162", "completion": "a = b \n y = a + 1"} +{"task_id": "HumanEval/163", "completion": "a = b \n y = a + 1"} diff --git a/scripts/evaluation/human_eval.jsonl_results.jsonl b/scripts/evaluation/human_eval.jsonl_results.jsonl new file mode 100644 index 0000000..8c14355 --- /dev/null +++ b/scripts/evaluation/human_eval.jsonl_results.jsonl @@ -0,0 +1,164 @@ +{"task_id": "HumanEval/0", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/1", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/2", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/3", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/4", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/5", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/6", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/7", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/8", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/9", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/10", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 21)", "passed": false} +{"task_id": "HumanEval/11", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/12", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/13", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/14", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 10)", "passed": false} +{"task_id": "HumanEval/15", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/16", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/17", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 18)", "passed": false} +{"task_id": "HumanEval/18", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/19", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/20", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/21", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/22", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/23", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/24", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 9)", "passed": false} +{"task_id": "HumanEval/25", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/26", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/27", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 9)", "passed": false} +{"task_id": "HumanEval/28", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/29", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/30", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/31", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 21)", "passed": false} +{"task_id": "HumanEval/32", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 25)", "passed": false} +{"task_id": "HumanEval/33", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/34", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 9)", "passed": false} +{"task_id": "HumanEval/35", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/36", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/37", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/38", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 19)", "passed": false} +{"task_id": "HumanEval/39", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 18)", "passed": false} +{"task_id": "HumanEval/40", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 21)", "passed": false} +{"task_id": "HumanEval/41", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/42", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/43", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 20)", "passed": false} +{"task_id": "HumanEval/44", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/45", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 9)", "passed": false} +{"task_id": "HumanEval/46", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 19)", "passed": false} +{"task_id": "HumanEval/47", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/48", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/49", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/50", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/51", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 20)", "passed": false} +{"task_id": "HumanEval/52", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/53", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/54", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 20)", "passed": false} +{"task_id": "HumanEval/55", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/56", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/57", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/58", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/59", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/60", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/61", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/62", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/63", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 18)", "passed": false} +{"task_id": "HumanEval/64", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 19)", "passed": false} +{"task_id": "HumanEval/65", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/66", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/67", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/68", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 37)", "passed": false} +{"task_id": "HumanEval/69", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/70", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/71", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/72", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 21)", "passed": false} +{"task_id": "HumanEval/73", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/74", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/75", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/76", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/77", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/78", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 21)", "passed": false} +{"task_id": "HumanEval/79", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/80", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/81", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 28)", "passed": false} +{"task_id": "HumanEval/82", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/83", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 8)", "passed": false} +{"task_id": "HumanEval/84", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/85", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 10)", "passed": false} +{"task_id": "HumanEval/86", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/87", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 23)", "passed": false} +{"task_id": "HumanEval/88", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 18)", "passed": false} +{"task_id": "HumanEval/89", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/90", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/91", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/92", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 21)", "passed": false} +{"task_id": "HumanEval/93", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/94", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/95", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/96", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/97", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/98", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/99", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 21)", "passed": false} +{"task_id": "HumanEval/100", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/101", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/102", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/103", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/104", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/105", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 25)", "passed": false} +{"task_id": "HumanEval/106", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/107", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 26)", "passed": false} +{"task_id": "HumanEval/108", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/109", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 30)", "passed": false} +{"task_id": "HumanEval/110", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/111", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/112", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/113", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/114", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/115", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 38)", "passed": false} +{"task_id": "HumanEval/116", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/117", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/118", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 18)", "passed": false} +{"task_id": "HumanEval/119", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 18)", "passed": false} +{"task_id": "HumanEval/120", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 28)", "passed": false} +{"task_id": "HumanEval/121", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/122", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/123", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 20)", "passed": false} +{"task_id": "HumanEval/124", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 23)", "passed": false} +{"task_id": "HumanEval/125", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/126", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 19)", "passed": false} +{"task_id": "HumanEval/127", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 23)", "passed": false} +{"task_id": "HumanEval/128", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/129", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 33)", "passed": false} +{"task_id": "HumanEval/130", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 20)", "passed": false} +{"task_id": "HumanEval/131", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/132", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/133", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/134", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/135", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/136", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/137", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/138", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 10)", "passed": false} +{"task_id": "HumanEval/139", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/140", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/141", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/142", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 16)", "passed": false} +{"task_id": "HumanEval/143", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 23)", "passed": false} +{"task_id": "HumanEval/144", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/145", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/146", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 11)", "passed": false} +{"task_id": "HumanEval/147", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 17)", "passed": false} +{"task_id": "HumanEval/148", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 19)", "passed": false} +{"task_id": "HumanEval/149", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 18)", "passed": false} +{"task_id": "HumanEval/150", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 12)", "passed": false} +{"task_id": "HumanEval/151", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 15)", "passed": false} +{"task_id": "HumanEval/152", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 18)", "passed": false} +{"task_id": "HumanEval/153", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 20)", "passed": false} +{"task_id": "HumanEval/154", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/155", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 10)", "passed": false} +{"task_id": "HumanEval/156", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/157", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/158", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} +{"task_id": "HumanEval/159", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 32)", "passed": false} +{"task_id": "HumanEval/160", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 28)", "passed": false} +{"task_id": "HumanEval/161", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 14)", "passed": false} +{"task_id": "HumanEval/162", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 10)", "passed": false} +{"task_id": "HumanEval/163", "completion": "a = b \n y = a + 1", "result": "failed: unexpected indent (, line 13)", "passed": false} diff --git a/scripts/evaluation/human_eval_bench.py b/scripts/evaluation/human_eval_bench.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/evaluation/metrics/bleu.py b/scripts/evaluation/metrics/bleu.py new file mode 100644 index 0000000..17a06ce --- /dev/null +++ b/scripts/evaluation/metrics/bleu.py @@ -0,0 +1,133 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# The following code is taken from CodeXGlue Repository - https://github.com/microsoft/CodeXGLUE/blob/main/Code-Code/code-to-code-trans/evaluator/CodeBLEU/bleu.py + + +"""Python implementation of BLEU and smooth-BLEU. + +This module provides a Python implementation of BLEU and smooth-BLEU. +Smooth BLEU is computed following the method outlined in the paper: +Chin-Yew Lin, Franz Josef Och. ORANGE: a method for evaluating automatic +evaluation metrics for machine translation. COLING 2004. +""" + +import collections +import math + + +def _get_ngrams(segment, max_order): + """Extracts all n-grams upto a given maximum order from an input segment. + + Args: + segment: text segment from which n-grams will be extracted. + max_order: maximum length in tokens of the n-grams returned by this + methods. + + Returns: + The Counter containing all n-grams upto max_order in segment + with a count of how many times each n-gram occurred. + """ + ngram_counts = collections.Counter() + for order in range(1, max_order + 1): + for i in range(0, len(segment) - order + 1): + ngram = tuple(segment[i : i + order]) + ngram_counts[ngram] += 1 + return ngram_counts + + +def compute_bleu(reference_corpus, translation_corpus, max_order=4, smooth=True): + """Computes BLEU score of translated segments against one or more references. + + Args: + reference_corpus: list of lists of references for each translation. Each + reference should be tokenized into a list of tokens. + translation_corpus: list of translations to score. Each translation + should be tokenized into a list of tokens. + max_order: Maximum n-gram order to use when computing BLEU score. + smooth: Whether or not to apply Lin et al. 2004 smoothing. + + Returns: + 3-Tuple with the BLEU score, n-gram precisions, geometric mean of n-gram + precisions and brevity penalty. + """ + matches_by_order = [0] * max_order + possible_matches_by_order = [0] * max_order + reference_length = 0 + translation_length = 0 + for (references, translation) in zip(reference_corpus, translation_corpus): + reference_length += min(len(r) for r in references) + translation_length += len(translation) + + merged_ref_ngram_counts = collections.Counter() + for reference in references: + merged_ref_ngram_counts |= _get_ngrams(reference, max_order) + translation_ngram_counts = _get_ngrams(translation, max_order) + overlap = translation_ngram_counts & merged_ref_ngram_counts + for ngram in overlap: + matches_by_order[len(ngram) - 1] += overlap[ngram] + for order in range(1, max_order + 1): + possible_matches = len(translation) - order + 1 + if possible_matches > 0: + possible_matches_by_order[order - 1] += possible_matches + + precisions = [0] * max_order + for i in range(0, max_order): + if smooth: + precisions[i] = (matches_by_order[i] + 1.0) / ( + possible_matches_by_order[i] + 1.0 + ) + else: + if possible_matches_by_order[i] > 0: + precisions[i] = ( + float(matches_by_order[i]) / possible_matches_by_order[i] + ) + else: + precisions[i] = 0.0 + + if min(precisions) > 0: + p_log_sum = sum((1.0 / max_order) * math.log(p) for p in precisions) + geo_mean = math.exp(p_log_sum) + else: + geo_mean = 0 + + ratio = float(translation_length) / reference_length + + if ratio > 1.0: + bp = 1.0 + else: + bp = math.exp(1 - 1.0 / ratio) + bleu = geo_mean * bp + bleu_score_dict = { + "bleu": bleu, + "precision": precisions, + "bp": bp, + "ratio": ratio, + "trans_len": translation_length, + "ref_len": reference_length, + } + return bleu_score_dict # (bleu, precisions, bp, ratio, translation_length, reference_length) + + +def bleu_test_case(): + """A simple functionality test case to evaluate BLEU""" + generated = [[["a", "=", "b", "\n", "y", "=", "a", "+", "1"]]] + reference = [["a", "=", "b", "\n", "print", "a"]] + score_dict = compute_bleu(generated, reference, smooth=False) + return score_dict + + +if __name__ == "__main__": + score_dict = bleu_test_case() + print(score_dict) diff --git a/scripts/evaluation/metrics/extrinsic_eval.py b/scripts/evaluation/metrics/extrinsic_eval.py new file mode 100644 index 0000000..d609870 --- /dev/null +++ b/scripts/evaluation/metrics/extrinsic_eval.py @@ -0,0 +1,46 @@ +from metrics.bleu import compute_bleu + + +def compute_exact_match(references, generated) -> float: + """ + Computes Exact Match Accuracy. + args: + reference: list of lists of references for each translation. Each + reference should be tokenized into a list of tokens. + translation: list of translations to score. Each translation + should be tokenized into a list of tokens. + returns: + exact_match_accuracy : Float + """ + assert ( + len(references[0]) == len(generated), + "Number of Samples should be equal in References and Synthesized Outputs..", + ) + exact_match_count = 0.0 + for gen, ref in zip(generated, references[0]): + if gen == ref: + exact_match_count += 1 + exact_match_acc = exact_match_count / len(generated) + return exact_match_acc + + +def compute_metrics(references, generated) -> dict: + """ + Calculates various metrics and returns the calculated dict of these matrics. + args: + reference: list of lists of references for each translation. Each + reference should be tokenized into a list of tokens. + translation: list of translations to score. Each translation + should be tokenized into a list of tokens. + returns: + A dicitonary with different metrics intact. + """ + metrics_dict = { + "smoothed_bleu_4": None, + "bleu_4": None, + "exact_match_acc": None, + } # Update as in new metrics are computed. + metrics_dict["smoothed_bleu_4"] = compute_bleu(references, generated, smooth=True) + metrics_dict["bleu_4"] = compute_bleu(references, generated, smooth=False) + metrics_dict["exact_match_acc"] = compute_exact_match(references, generated) + return metrics_dict From 73aa110de905940eaed1fe83e6834bf5feb2dc5b Mon Sep 17 00:00:00 2001 From: ncoop57 Date: Mon, 12 Jul 2021 13:15:22 -0400 Subject: [PATCH 09/15] Add model card and data sheet templates --- docs/DATASHEET.md | 200 ++++++++++++++++++++++++++++++++++++++++++++++ docs/MODELCARD.md | 98 +++++++++++++++++++++++ 2 files changed, 298 insertions(+) create mode 100644 docs/DATASHEET.md create mode 100644 docs/MODELCARD.md diff --git a/docs/DATASHEET.md b/docs/DATASHEET.md new file mode 100644 index 0000000..33190c8 --- /dev/null +++ b/docs/DATASHEET.md @@ -0,0 +1,200 @@ +--- +YAML tags: +- copy-paste the tags obtained with the tagging app: https://github.com/huggingface/datasets-tagging +--- + +# Dataset Card Creation Guide + +## Table of Contents +- [Dataset Card Creation Guide](#dataset-card-creation-guide) + - [Table of Contents](#table-of-contents) + - [Dataset Description](#dataset-description) + - [Dataset Summary](#dataset-summary) + - [Supported Tasks and Leaderboards](#supported-tasks-and-leaderboards) + - [Languages](#languages) + - [Dataset Structure](#dataset-structure) + - [Data Instances](#data-instances) + - [Data Fields](#data-fields) + - [Data Splits](#data-splits) + - [Dataset Creation](#dataset-creation) + - [Curation Rationale](#curation-rationale) + - [Source Data](#source-data) + - [Initial Data Collection and Normalization](#initial-data-collection-and-normalization) + - [Who are the source language producers?](#who-are-the-source-language-producers) + - [Annotations](#annotations) + - [Annotation process](#annotation-process) + - [Who are the annotators?](#who-are-the-annotators) + - [Personal and Sensitive Information](#personal-and-sensitive-information) + - [Considerations for Using the Data](#considerations-for-using-the-data) + - [Social Impact of Dataset](#social-impact-of-dataset) + - [Discussion of Biases](#discussion-of-biases) + - [Other Known Limitations](#other-known-limitations) + - [Additional Information](#additional-information) + - [Dataset Curators](#dataset-curators) + - [Licensing Information](#licensing-information) + - [Citation Information](#citation-information) + - [Contributions](#contributions) + +## Dataset Description + +- **Homepage:** [Add homepage URL here if available (unless it's a GitHub repository)]() +- **Repository:** [If the dataset is hosted on github or has a github homepage, add URL here]() +- **Paper:** [If the dataset was introduced by a paper or there was a paper written describing the dataset, add URL here (landing page for Arxiv paper preferred)]() +- **Leaderboard:** [If the dataset supports an active leaderboard, add link here]() +- **Point of Contact:** [If known, name and email of at least one person the reader can contact for questions about the dataset.]() + +### Dataset Summary + +Briefly summarize the dataset, its intended use and the supported tasks. Give an overview of how and why the dataset was created. The summary should explicitly mention the languages present in the dataset (possibly in broad terms, e.g. *translations between several pairs of European languages*), and describe the domain, topic, or genre covered. + +### Supported Tasks and Leaderboards + +For each of the tasks tagged for this dataset, give a brief description of the tag, metrics, and suggested models (with a link to their HuggingFace implementation if available). Give a similar description of tasks that were not covered by the structured tag set (repace the `task-category-tag` with an appropriate `other:other-task-name`). + +- `task-category-tag`: The dataset can be used to train a model for [TASK NAME], which consists in [TASK DESCRIPTION]. Success on this task is typically measured by achieving a *high/low* [metric name](https://huggingface.co/metrics/metric_name). The ([model name](https://huggingface.co/model_name) or [model class](https://huggingface.co/transformers/model_doc/model_class.html)) model currently achieves the following score. *[IF A LEADERBOARD IS AVAILABLE]:* This task has an active leaderboard which can be found at [leaderboard url]() and ranks models based on [metric name](https://huggingface.co/metrics/metric_name) while also reporting [other metric name](https://huggingface.co/metrics/other_metric_name). + +### Languages + +Provide a brief overview of the languages represented in the dataset. Describe relevant details about specifics of the language such as whether it is social media text, African American English,... + +When relevant, please provide [BCP-47 codes](https://tools.ietf.org/html/bcp47), which consist of a [primary language subtag](https://tools.ietf.org/html/bcp47#section-2.2.1), with a [script subtag](https://tools.ietf.org/html/bcp47#section-2.2.3) and/or [region subtag](https://tools.ietf.org/html/bcp47#section-2.2.4) if available. + +## Dataset Structure + +### Data Instances + +Provide an JSON-formatted example and brief description of a typical instance in the dataset. If available, provide a link to further examples. + +``` +{ + 'example_field': ..., + ... +} +``` + +Provide any additional information that is not covered in the other sections about the data here. In particular describe any relationships between data points and if these relationships are made explicit. + +### Data Fields + +List and describe the fields present in the dataset. Mention their data type, and whether they are used as input or output in any of the tasks the dataset currently supports. If the data has span indices, describe their attributes, such as whether they are at the character level or word level, whether they are contiguous or not, etc. If the datasets contains example IDs, state whether they have an inherent meaning, such as a mapping to other datasets or pointing to relationships between data points. + +- `example_field`: description of `example_field` + +Note that the descriptions can be initialized with the **Show Markdown Data Fields** output of the [tagging app](https://github.com/huggingface/datasets-tagging), you will then only need to refine the generated descriptions. + +### Data Splits + +Describe and name the splits in the dataset if there are more than one. + +Describe any criteria for splitting the data, if used. If their are differences between the splits (e.g. if the training annotations are machine-generated and the dev and test ones are created by humans, or if different numbers of annotators contributed to each example), describe them here. + +Provide the sizes of each split. As appropriate, provide any descriptive statistics for the features, such as average length. For example: + +| | Tain | Valid | Test | +| ----- | ------ | ----- | ---- | +| Input Sentences | | | | +| Average Sentence Length | | | | + +## Dataset Creation + +### Curation Rationale + +What need motivated the creation of this dataset? What are some of the reasons underlying the major choices involved in putting it together? + +### Source Data + +This section describes the source data (e.g. news text and headlines, social media posts, translated sentences,...) + +#### Initial Data Collection and Normalization + +Describe the data collection process. Describe any criteria for data selection or filtering. List any key words or search terms used. If possible, include runtime information for the collection process. + +If data was collected from other pre-existing datasets, link to source here and to their [Hugging Face version](https://huggingface.co/datasets/dataset_name). + +If the data was modified or normalized after being collected (e.g. if the data is word-tokenized), describe the process and the tools used. + +#### Who are the source language producers? + +State whether the data was produced by humans or machine generated. Describe the people or systems who originally created the data. + +If available, include self-reported demographic or identity information for the source data creators, but avoid inferring this information. Instead state that this information is unknown. See [Larson 2017](https://www.aclweb.org/anthology/W17-1601.pdf) for using identity categories as a variables, particularly gender. + +Describe the conditions under which the data was created (for example, if the producers were crowdworkers, state what platform was used, or if the data was found, what website the data was found on). If compensation was provided, include that information here. + +Describe other people represented or mentioned in the data. Where possible, link to references for the information. + +### Annotations + +If the dataset contains annotations which are not part of the initial data collection, describe them in the following paragraphs. + +#### Annotation process + +If applicable, describe the annotation process and any tools used, or state otherwise. Describe the amount of data annotated, if not all. Describe or reference annotation guidelines provided to the annotators. If available, provide interannotator statistics. Describe any annotation validation processes. + +#### Who are the annotators? + +If annotations were collected for the source data (such as class labels or syntactic parses), state whether the annotations were produced by humans or machine generated. + +Describe the people or systems who originally created the annotations and their selection criteria if applicable. + +If available, include self-reported demographic or identity information for the annotators, but avoid inferring this information. Instead state that this information is unknown. See [Larson 2017](https://www.aclweb.org/anthology/W17-1601.pdf) for using identity categories as a variables, particularly gender. + +Describe the conditions under which the data was annotated (for example, if the annotators were crowdworkers, state what platform was used, or if the data was found, what website the data was found on). If compensation was provided, include that information here. + +### Personal and Sensitive Information + +State whether the dataset uses identity categories and, if so, how the information is used. Describe where this information comes from (i.e. self-reporting, collecting from profiles, inferring, etc.). See [Larson 2017](https://www.aclweb.org/anthology/W17-1601.pdf) for using identity categories as a variables, particularly gender. State whether the data is linked to individuals and whether those individuals can be identified in the dataset, either directly or indirectly (i.e., in combination with other data). + +State whether the dataset contains other data that might be considered sensitive (e.g., data that reveals racial or ethnic origins, sexual orientations, religious beliefs, political opinions or union memberships, or locations; financial or health data; biometric or genetic data; forms of government identification, such as social security numbers; criminal history). + +If efforts were made to anonymize the data, describe the anonymization process. + +## Considerations for Using the Data + +### Social Impact of Dataset + +Please discuss some of the ways you believe the use of this dataset will impact society. + +The statement should include both positive outlooks, such as outlining how technologies developed through its use may improve people's lives, and discuss the accompanying risks. These risks may range from making important decisions more opaque to people who are affected by the technology, to reinforcing existing harmful biases (whose specifics should be discussed in the next section), among other considerations. + +Also describe in this section if the proposed dataset contains a low-resource or under-represented language. If this is the case or if this task has any impact on underserved communities, please elaborate here. + +### Discussion of Biases + +Provide descriptions of specific biases that are likely to be reflected in the data, and state whether any steps were taken to reduce their impact. + +For Wikipedia text, see for example [Dinan et al 2020 on biases in Wikipedia (esp. Table 1)](https://arxiv.org/abs/2005.00614), or [Blodgett et al 2020](https://www.aclweb.org/anthology/2020.acl-main.485/) for a more general discussion of the topic. + +If analyses have been run quantifying these biases, please add brief summaries and links to the studies here. + +### Other Known Limitations + +If studies of the datasets have outlined other limitations of the dataset, such as annotation artifacts, please outline and cite them here. + +## Additional Information + +### Dataset Curators + +List the people involved in collecting the dataset and their affiliation(s). If funding information is known, include it here. + +### Licensing Information + +Provide the license and link to the license webpage if available. + +### Citation Information + +Provide the [BibTex](http://www.bibtex.org/)-formatted reference for the dataset. For example: +``` +@article{article_id, + author = {Author List}, + title = {Dataset Paper Title}, + journal = {Publication Venue}, + year = {2525} +} +``` + +If the dataset has a [DOI](https://www.doi.org/), please provide it here. + +### Contributions + +Thanks to [@github-username](https://github.com/) for adding this dataset. diff --git a/docs/MODELCARD.md b/docs/MODELCARD.md new file mode 100644 index 0000000..a94680c --- /dev/null +++ b/docs/MODELCARD.md @@ -0,0 +1,98 @@ +--- +language: +- ru +- en +thumbnail: https://raw.githubusercontent.com/JetRunner/BERT-of-Theseus/master/bert-of-theseus.png +tags: +- translation +- fsmt +license: Apache 2.0 +datasets: +- wmt19 +metrics: +- bleu +- sacrebleu +--- + +# MyModel + +## Model description + +This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for {src_lang}-{tgt_lang}. + +For more details, please see, [Facebook FAIR's WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616). + +The abbreviation FSMT stands for FairSeqMachineTranslation + +All four models are available: + +* [wmt19-en-ru](https://huggingface.co/facebook/wmt19-en-ru) +* [wmt19-ru-en](https://huggingface.co/facebook/wmt19-ru-en) +* [wmt19-en-de](https://huggingface.co/facebook/wmt19-en-de) +* [wmt19-de-en](https://huggingface.co/facebook/wmt19-de-en) + +## Intended uses & limitations + +#### How to use + +```python +from transformers.tokenization_fsmt import FSMTTokenizer +from transformers.modeling_fsmt import FSMTForConditionalGeneration +mname = "facebook/wmt19-ru-en" +tokenizer = FSMTTokenizer.from_pretrained(mname) +model = FSMTForConditionalGeneration.from_pretrained(mname) + +input = "Машинное обучение - это здорово, не так ли?" +input_ids = tokenizer.encode(input, return_tensors="pt") +outputs = model.generate(input_ids) +decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) +print(decoded) # Machine learning is great, isn't it? +``` + +#### Limitations and bias + +- The original (and this ported model) doesn't seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981) + + +## Training data + +Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616). + + +## Training procedure + + +## Eval results + +pair | fairseq | transformers +-------|---------|---------- +ru-en | [41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937) | 39.20 + + +The score was calculated using this code: + +```bash +git clone https://github.com/huggingface/transformers +cd transformers +export PAIR=ru-en +export DATA_DIR=data/$PAIR +export SAVE_DIR=data/$PAIR +export BS=8 +export NUM_BEAMS=15 +mkdir -p $DATA_DIR +sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source +sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target +echo $PAIR +PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS +``` + +### BibTeX entry and citation info + +```bibtex +@inproceedings{..., + year={2020}, + title={Facebook FAIR's WMT19 News Translation Task Submission}, + author={Ng, Nathan and Yee, Kyra and Baevski, Alexei and Ott, Myle and Auli, Michael and Edunov, Sergey}, + booktitle={Proc. of WMT}, +} +``` From 745372a744df314ed0bfcb601732dd94da8efc65 Mon Sep 17 00:00:00 2001 From: ncoop57 Date: Wed, 14 Jul 2021 20:12:26 -0400 Subject: [PATCH 10/15] SOme initial clean up --- ...ng_on_TPU_with_🤗_Transformers_&_JAX.ipynb | 7182 ----------------- EDA.ipynb | 208 - LICENSE | 2 +- .../check_repo_vulnerabilities.py | 0 .../convert_to_gh_downloader_format.py | 0 .../deduplication}/README.md | 0 .../deduplication}/deduplication.py | 0 .../deduplication}/deduplication_parallel.py | 0 .../deduplication}/deduplication_streaming.py | 0 .../deduplication}/script.py | 0 {scripts => data_processing}/download_data.sh | 0 {scripts => data_processing}/scorecard | Bin {nbs => evaluation}/data_processing.ipynb | 0 {nbs => evaluation}/gh-data-exploration.ipynb | 0 {metrics => evaluation/metrics}/bleu.py | 0 .../metrics}/extrinsic_eval.py | 0 flax-gpt-neo-clm-v2.ipynb | 6554 --------------- flax-gpt-neo-clm-v3.ipynb | 837 -- flax-gpt-neo-clm.ipynb | 6626 --------------- github-scraping/01_metadata_by_github_api.py | 91 - github-scraping/02_clone_repos.sh | 10 - github-scraping/03_lexer.py | 50 - github-scraping/README.md | 7 - gpt-neo-test.ipynb | 487 -- ...quirements.txt => requirements_scripts.txt | 0 run_clm_gpt_neo.sh | 32 - run_clm_streaming.sh | 37 - run_clm_streaming_flax.py | 885 -- run_clm_streaming_wikitext.sh | 37 - run_clm_wikitext.sh | 35 - scripts/notify_repo_owners.py | 45 - scripts/read_repo_replies.py | 27 - {scripts => training}/add_new_tokens.py | 0 code_clippy.py => training/code_clippy.py | 0 {scripts => training}/new_tokens.json | 0 run_clm_flax.py => training/run_clm_flax.py | 0 .../run_clm_gpt_neo_13b.sh | 0 .../run_clm_gpt_neo_27b.sh | 0 .../run_clm_streaming_125m_1e-4lr_1024bs.sh | 0 .../run_clm_streaming_1_3b_1e-4lr_1024bs.sh | 0 .../run_clm_streaming_flax_clean.py | 0 .../run_clm_streaming_flax_v2.py | 0 utils.py => training/utils.py | 0 43 files changed, 1 insertion(+), 23151 deletions(-) delete mode 100644 Causal_Language_Model_Training_on_TPU_with_🤗_Transformers_&_JAX.ipynb delete mode 100644 EDA.ipynb rename {scripts => data_processing}/check_repo_vulnerabilities.py (100%) rename {scripts => data_processing}/convert_to_gh_downloader_format.py (100%) rename {deduplication => data_processing/deduplication}/README.md (100%) rename {deduplication => data_processing/deduplication}/deduplication.py (100%) rename {deduplication => data_processing/deduplication}/deduplication_parallel.py (100%) rename {deduplication => data_processing/deduplication}/deduplication_streaming.py (100%) rename {deduplication => data_processing/deduplication}/script.py (100%) rename {scripts => data_processing}/download_data.sh (100%) rename {scripts => data_processing}/scorecard (100%) rename {nbs => evaluation}/data_processing.ipynb (100%) rename {nbs => evaluation}/gh-data-exploration.ipynb (100%) rename {metrics => evaluation/metrics}/bleu.py (100%) rename {metrics => evaluation/metrics}/extrinsic_eval.py (100%) delete mode 100644 flax-gpt-neo-clm-v2.ipynb delete mode 100644 flax-gpt-neo-clm-v3.ipynb delete mode 100644 flax-gpt-neo-clm.ipynb delete mode 100644 github-scraping/01_metadata_by_github_api.py delete mode 100755 github-scraping/02_clone_repos.sh delete mode 100644 github-scraping/03_lexer.py delete mode 100644 github-scraping/README.md delete mode 100644 gpt-neo-test.ipynb rename scripts/requirements.txt => requirements_scripts.txt (100%) delete mode 100644 run_clm_gpt_neo.sh delete mode 100644 run_clm_streaming.sh delete mode 100755 run_clm_streaming_flax.py delete mode 100644 run_clm_streaming_wikitext.sh delete mode 100644 run_clm_wikitext.sh delete mode 100644 scripts/notify_repo_owners.py delete mode 100644 scripts/read_repo_replies.py rename {scripts => training}/add_new_tokens.py (100%) rename code_clippy.py => training/code_clippy.py (100%) rename {scripts => training}/new_tokens.json (100%) rename run_clm_flax.py => training/run_clm_flax.py (100%) rename run_clm_gpt_neo_13b.sh => training/run_clm_gpt_neo_13b.sh (100%) rename run_clm_gpt_neo_27b.sh => training/run_clm_gpt_neo_27b.sh (100%) rename run_clm_streaming_125m_1e-4lr_1024bs.sh => training/run_clm_streaming_125m_1e-4lr_1024bs.sh (100%) rename run_clm_streaming_1_3b_1e-4lr_1024bs.sh => training/run_clm_streaming_1_3b_1e-4lr_1024bs.sh (100%) rename run_clm_streaming_flax_clean.py => training/run_clm_streaming_flax_clean.py (100%) rename run_clm_streaming_flax_v2.py => training/run_clm_streaming_flax_v2.py (100%) rename utils.py => training/utils.py (100%) diff --git a/Causal_Language_Model_Training_on_TPU_with_🤗_Transformers_&_JAX.ipynb b/Causal_Language_Model_Training_on_TPU_with_🤗_Transformers_&_JAX.ipynb deleted file mode 100644 index b3d84d9..0000000 --- a/Causal_Language_Model_Training_on_TPU_with_🤗_Transformers_&_JAX.ipynb +++ /dev/null @@ -1,7182 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "accelerator": "TPU", - "colab": { - "name": "Causal Language Model Training on TPU with 🤗 Transformers & JAX", - "provenance": [], - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "8c156c360afb4ddc962b87577e093cc4": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_view_name": "HBoxView", - "_dom_classes": [], - "_model_name": "HBoxModel", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.5.0", - "box_style": "", - "layout": "IPY_MODEL_9808f288735b4f5d9eea7377d4603ccc", - "_model_module": "@jupyter-widgets/controls", - "children": [ - "IPY_MODEL_68681edff0db43559012b6653571a289", - "IPY_MODEL_125e8d57908f4679824c96d5247f7119" - ] - } - }, - "9808f288735b4f5d9eea7377d4603ccc": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "68681edff0db43559012b6653571a289": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_view_name": "ProgressView", - "style": "IPY_MODEL_8134ef53fa8044c99b537386e88dbfc9", - "_dom_classes": [], - "description": "Epoch ...: 100%", - "_model_name": "FloatProgressModel", - "bar_style": "success", - "max": 10, - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": 10, - "_view_count": null, - "_view_module_version": "1.5.0", - "orientation": "horizontal", - "min": 0, - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_9b35ba10c85048ce8044b87ab9948611" - } - }, - "125e8d57908f4679824c96d5247f7119": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_view_name": "HTMLView", - "style": "IPY_MODEL_e246db4008b14a23b55d2820be3b1ed8", - "_dom_classes": [], - "description": "", - "_model_name": "HTMLModel", - "placeholder": "​", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": " 10/10 [19:05<00:00, 114.56s/it]", - "_view_count": null, - "_view_module_version": "1.5.0", - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_b08557f6936e4af9a23f659649979c8d" - } - }, - "8134ef53fa8044c99b537386e88dbfc9": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "ProgressStyleModel", - "description_width": "initial", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "bar_color": null, - "_model_module": "@jupyter-widgets/controls" - } - }, - "9b35ba10c85048ce8044b87ab9948611": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "e246db4008b14a23b55d2820be3b1ed8": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "DescriptionStyleModel", - "description_width": "", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "_model_module": "@jupyter-widgets/controls" - } - }, - "b08557f6936e4af9a23f659649979c8d": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "dac789bcf81e4aac86717438ed6e8bbe": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_view_name": "HBoxView", - "_dom_classes": [], - "_model_name": "HBoxModel", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.5.0", - "box_style": "", - "layout": "IPY_MODEL_a1452333389441eab2c8016cd019d2f5", - "_model_module": "@jupyter-widgets/controls", - "children": [ - "IPY_MODEL_799b382dd6c74d528cf77813f82a7335", - "IPY_MODEL_51d0dfb67bbd47ff88964bb446a91967" - ] - } - }, - "a1452333389441eab2c8016cd019d2f5": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "799b382dd6c74d528cf77813f82a7335": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_view_name": "ProgressView", - "style": "IPY_MODEL_940ce122551e42509e8cec0659e47c77", - "_dom_classes": [], - "description": "Training...: 100%", - "_model_name": "FloatProgressModel", - "bar_style": "", - "max": 137, - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": 137, - "_view_count": null, - "_view_module_version": "1.5.0", - "orientation": "horizontal", - "min": 0, - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_a4234dc426d3459a93263c8b5fe5a34b" - } - }, - "51d0dfb67bbd47ff88964bb446a91967": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_view_name": "HTMLView", - "style": "IPY_MODEL_3549bf51a354408b9a39cb4532957617", - "_dom_classes": [], - "description": "", - "_model_name": "HTMLModel", - "placeholder": "​", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": " 137/137 [03:28<00:00, 1.41it/s]", - "_view_count": null, - "_view_module_version": "1.5.0", - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_7f05c774880548d2b73e69fb847c0a2e" - } - }, - "940ce122551e42509e8cec0659e47c77": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "ProgressStyleModel", - "description_width": "initial", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "bar_color": null, - "_model_module": "@jupyter-widgets/controls" - } - }, - "a4234dc426d3459a93263c8b5fe5a34b": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "3549bf51a354408b9a39cb4532957617": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "DescriptionStyleModel", - "description_width": "", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "_model_module": "@jupyter-widgets/controls" - } - }, - "7f05c774880548d2b73e69fb847c0a2e": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "71fa85dc266149e28685b1f5252185a5": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_view_name": "HBoxView", - "_dom_classes": [], - "_model_name": "HBoxModel", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.5.0", - "box_style": "", - "layout": "IPY_MODEL_9cb943192a154dbc8c7961f128449bbd", - "_model_module": "@jupyter-widgets/controls", - "children": [ - "IPY_MODEL_9de6ea64b67c443fbce1fd6aef4a7409", - "IPY_MODEL_9658cc6977bb4fdf8a081aa99c300872" - ] - } - }, - "9cb943192a154dbc8c7961f128449bbd": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "9de6ea64b67c443fbce1fd6aef4a7409": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_view_name": "ProgressView", - "style": "IPY_MODEL_07b26680343f42fc805a3b52b398a5f2", - "_dom_classes": [], - "description": "Evaluation...: 100%", - "_model_name": "FloatProgressModel", - "bar_style": "", - "max": 12, - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": 12, - "_view_count": null, - "_view_module_version": "1.5.0", - "orientation": "horizontal", - "min": 0, - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_97216855bb7f483882afae68bf78a51d" - } - }, - "9658cc6977bb4fdf8a081aa99c300872": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_view_name": "HTMLView", - "style": "IPY_MODEL_a1b9aa80b73748f18e3938df073b3d22", - "_dom_classes": [], - "description": "", - "_model_name": "HTMLModel", - "placeholder": "​", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": " 12/12 [00:12<00:00, 1.78it/s]", - "_view_count": null, - "_view_module_version": "1.5.0", - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_ce70b524ffab4052912d47b8adee6748" - } - }, - "07b26680343f42fc805a3b52b398a5f2": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "ProgressStyleModel", - "description_width": "initial", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "bar_color": null, - "_model_module": "@jupyter-widgets/controls" - } - }, - "97216855bb7f483882afae68bf78a51d": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "a1b9aa80b73748f18e3938df073b3d22": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "DescriptionStyleModel", - "description_width": "", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "_model_module": "@jupyter-widgets/controls" - } - }, - "ce70b524ffab4052912d47b8adee6748": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "3c5df3c942ca4c768926b431064be351": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_view_name": "HBoxView", - "_dom_classes": [], - "_model_name": "HBoxModel", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.5.0", - "box_style": "", - "layout": "IPY_MODEL_f2e66aebdb4d4812a1f61c1ff990390a", - "_model_module": "@jupyter-widgets/controls", - "children": [ - "IPY_MODEL_269cc543a69e4ba3bae7f0bef67f1e25", - "IPY_MODEL_8cb8398bb3c94284ac2ff42c7b3bc7c5" - ] - } - }, - "f2e66aebdb4d4812a1f61c1ff990390a": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "269cc543a69e4ba3bae7f0bef67f1e25": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_view_name": "ProgressView", - "style": "IPY_MODEL_0b898451f79c4b6eb1678a8f366a5cbf", - "_dom_classes": [], - "description": "Training...: 100%", - "_model_name": "FloatProgressModel", - "bar_style": "", - "max": 137, - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": 137, - "_view_count": null, - "_view_module_version": "1.5.0", - "orientation": "horizontal", - "min": 0, - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_7a5426d6898e4ba2b5e3c48a65096232" - } - }, - "8cb8398bb3c94284ac2ff42c7b3bc7c5": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_view_name": "HTMLView", - "style": "IPY_MODEL_2cf7d4c7dc6f413eb8cc9460e5de8d93", - "_dom_classes": [], - "description": "", - "_model_name": "HTMLModel", - "placeholder": "​", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": " 137/137 [01:37<00:00, 1.41it/s]", - "_view_count": null, - "_view_module_version": "1.5.0", - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_f7851bd9702949ee8e6278ad69c9d981" - } - }, - "0b898451f79c4b6eb1678a8f366a5cbf": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "ProgressStyleModel", - "description_width": "initial", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "bar_color": null, - "_model_module": "@jupyter-widgets/controls" - } - }, - "7a5426d6898e4ba2b5e3c48a65096232": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "2cf7d4c7dc6f413eb8cc9460e5de8d93": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "DescriptionStyleModel", - "description_width": "", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "_model_module": "@jupyter-widgets/controls" - } - }, - "f7851bd9702949ee8e6278ad69c9d981": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "ab08bd64073d4d488f9c41e8595bac08": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_view_name": "HBoxView", - "_dom_classes": [], - "_model_name": "HBoxModel", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.5.0", - "box_style": "", - "layout": "IPY_MODEL_5e1805a3c7f24c3f9f15e86d15462d44", - "_model_module": "@jupyter-widgets/controls", - "children": [ - "IPY_MODEL_fc2e1a72620742499457ab2e31da12a9", - "IPY_MODEL_b3b4d96626044375bad11ba48cc25ee3" - ] - } - }, - "5e1805a3c7f24c3f9f15e86d15462d44": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "fc2e1a72620742499457ab2e31da12a9": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_view_name": "ProgressView", - "style": "IPY_MODEL_c55e4e5f6d0a42b4ac25fb6d68a8e842", - "_dom_classes": [], - "description": "Evaluation...: 100%", - "_model_name": "FloatProgressModel", - "bar_style": "", - "max": 12, - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": 12, - "_view_count": null, - "_view_module_version": "1.5.0", - "orientation": "horizontal", - "min": 0, - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_c7cca560cd4340bb986297655f513746" - } - }, - "b3b4d96626044375bad11ba48cc25ee3": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_view_name": "HTMLView", - "style": "IPY_MODEL_cd4892863d524637b8e567e2beed23bb", - "_dom_classes": [], - "description": "", - "_model_name": "HTMLModel", - "placeholder": "​", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": " 12/12 [00:05<00:00, 2.31it/s]", - "_view_count": null, - "_view_module_version": "1.5.0", - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_c85760cb6ccb4020be999357f2501bf9" - } - }, - "c55e4e5f6d0a42b4ac25fb6d68a8e842": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "ProgressStyleModel", - "description_width": "initial", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "bar_color": null, - "_model_module": "@jupyter-widgets/controls" - } - }, - "c7cca560cd4340bb986297655f513746": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "cd4892863d524637b8e567e2beed23bb": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "DescriptionStyleModel", - "description_width": "", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "_model_module": "@jupyter-widgets/controls" - } - }, - "c85760cb6ccb4020be999357f2501bf9": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "7d0775fcf6a8497db20326dc94bd2739": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_view_name": "HBoxView", - "_dom_classes": [], - "_model_name": "HBoxModel", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.5.0", - "box_style": "", - "layout": "IPY_MODEL_5215d791d9144c089b6996336c3085cd", - "_model_module": "@jupyter-widgets/controls", - "children": [ - "IPY_MODEL_839daa7d5cdc4669942f1b5ef5bf4903", - "IPY_MODEL_4bc03fce15a14355b3429c138ea1eadd" - ] - } - }, - "5215d791d9144c089b6996336c3085cd": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "839daa7d5cdc4669942f1b5ef5bf4903": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_view_name": "ProgressView", - "style": "IPY_MODEL_d7cddd2cafe84d17829096b31ee5e93e", - "_dom_classes": [], - "description": "Training...: 100%", - "_model_name": "FloatProgressModel", - "bar_style": "", - "max": 137, - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": 137, - "_view_count": null, - "_view_module_version": "1.5.0", - "orientation": "horizontal", - "min": 0, - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_b4ce8892d5784011a649eeca7b17fd0d" - } - }, - "4bc03fce15a14355b3429c138ea1eadd": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_view_name": "HTMLView", - "style": "IPY_MODEL_7a7b598979174b338f8fdfc90af61e47", - "_dom_classes": [], - "description": "", - "_model_name": "HTMLModel", - "placeholder": "​", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": " 137/137 [01:37<00:00, 1.41it/s]", - "_view_count": null, - "_view_module_version": "1.5.0", - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_cd946e84023445d59eea9893fba02f1a" - } - }, - "d7cddd2cafe84d17829096b31ee5e93e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "ProgressStyleModel", - "description_width": "initial", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "bar_color": null, - "_model_module": "@jupyter-widgets/controls" - } - }, - "b4ce8892d5784011a649eeca7b17fd0d": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "7a7b598979174b338f8fdfc90af61e47": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "DescriptionStyleModel", - "description_width": "", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "_model_module": "@jupyter-widgets/controls" - } - }, - "cd946e84023445d59eea9893fba02f1a": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "4ec581224b3f47f2acdfaeb740c94526": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_view_name": "HBoxView", - "_dom_classes": [], - "_model_name": "HBoxModel", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.5.0", - "box_style": "", - "layout": "IPY_MODEL_81c25bb635db4bc9a62636bf3a3868eb", - "_model_module": "@jupyter-widgets/controls", - "children": [ - "IPY_MODEL_1462263d23c340e781d596a3f5414e03", - "IPY_MODEL_c8bd20a456cc4783a2fbbdec9a2880ea" - ] - } - }, - "81c25bb635db4bc9a62636bf3a3868eb": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "1462263d23c340e781d596a3f5414e03": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_view_name": "ProgressView", - "style": "IPY_MODEL_f0aa35a704a84967a3471e5f698bb04c", - "_dom_classes": [], - "description": "Evaluation...: 100%", - "_model_name": "FloatProgressModel", - "bar_style": "", - "max": 12, - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": 12, - "_view_count": null, - "_view_module_version": "1.5.0", - "orientation": "horizontal", - "min": 0, - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_f2c541b4ecd04729a518414f8b878099" - } - }, - "c8bd20a456cc4783a2fbbdec9a2880ea": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_view_name": "HTMLView", - "style": "IPY_MODEL_402795b6506a45c9831fb0ddb2ce0749", - "_dom_classes": [], - "description": "", - "_model_name": "HTMLModel", - "placeholder": "​", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": " 12/12 [00:05<00:00, 2.35it/s]", - "_view_count": null, - "_view_module_version": "1.5.0", - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_0ba00c3340414029938c9c947d4a74b6" - } - }, - "f0aa35a704a84967a3471e5f698bb04c": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "ProgressStyleModel", - "description_width": "initial", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "bar_color": null, - "_model_module": "@jupyter-widgets/controls" - } - }, - "f2c541b4ecd04729a518414f8b878099": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "402795b6506a45c9831fb0ddb2ce0749": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "DescriptionStyleModel", - "description_width": "", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "_model_module": "@jupyter-widgets/controls" - } - }, - "0ba00c3340414029938c9c947d4a74b6": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "d17f610741f142b782c74ac4e04e6481": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_view_name": "HBoxView", - "_dom_classes": [], - "_model_name": "HBoxModel", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.5.0", - "box_style": "", - "layout": "IPY_MODEL_84c8a7cbfdb04690941f8c32299744bb", - "_model_module": "@jupyter-widgets/controls", - "children": [ - "IPY_MODEL_f3f0b212907843f5be2131b7b60a955d", - "IPY_MODEL_0a6a241c99bb49ad9baaef052b157964" - ] - } - }, - "84c8a7cbfdb04690941f8c32299744bb": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "f3f0b212907843f5be2131b7b60a955d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_view_name": "ProgressView", - "style": "IPY_MODEL_b642c546813240a499a165279c743e7a", - "_dom_classes": [], - "description": "Training...: 100%", - "_model_name": "FloatProgressModel", - "bar_style": "", - "max": 137, - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": 137, - "_view_count": null, - "_view_module_version": "1.5.0", - "orientation": "horizontal", - "min": 0, - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_6acb415dc8b947bb85ab8f8959bc10e1" - } - }, - "0a6a241c99bb49ad9baaef052b157964": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_view_name": "HTMLView", - "style": "IPY_MODEL_2c311c4506264995b7fbaba02333e747", - "_dom_classes": [], - "description": "", - "_model_name": "HTMLModel", - "placeholder": "​", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": " 137/137 [01:38<00:00, 1.40it/s]", - "_view_count": null, - "_view_module_version": "1.5.0", - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_2fde17ab63e5444799ec1dc5501d946e" - } - }, - "b642c546813240a499a165279c743e7a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "ProgressStyleModel", - "description_width": "initial", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "bar_color": null, - "_model_module": "@jupyter-widgets/controls" - } - }, - "6acb415dc8b947bb85ab8f8959bc10e1": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "2c311c4506264995b7fbaba02333e747": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "DescriptionStyleModel", - "description_width": "", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "_model_module": "@jupyter-widgets/controls" - } - }, - "2fde17ab63e5444799ec1dc5501d946e": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "6173f15a3e5a40819bd4f3833544fd78": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_view_name": "HBoxView", - "_dom_classes": [], - "_model_name": "HBoxModel", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.5.0", - "box_style": "", - "layout": "IPY_MODEL_da2f69c01bcd4139980a84ddca297817", - "_model_module": "@jupyter-widgets/controls", - "children": [ - "IPY_MODEL_b9470519dfff45228e54d0181337ee17", - "IPY_MODEL_b5824ca248b74052bb1696b47369838e" - ] - } - }, - "da2f69c01bcd4139980a84ddca297817": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "b9470519dfff45228e54d0181337ee17": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_view_name": "ProgressView", - "style": "IPY_MODEL_11f8b1ddd5af4bb3babd492bbaf3fb7d", - "_dom_classes": [], - "description": "Evaluation...: 100%", - "_model_name": "FloatProgressModel", - "bar_style": "", - "max": 12, - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": 12, - "_view_count": null, - "_view_module_version": "1.5.0", - "orientation": "horizontal", - "min": 0, - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_861e2a6f8e3e4a5cacfd0a746180cc90" - } - }, - "b5824ca248b74052bb1696b47369838e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_view_name": "HTMLView", - "style": "IPY_MODEL_181f0680e5bb4d23a69f9cc747207883", - "_dom_classes": [], - "description": "", - "_model_name": "HTMLModel", - "placeholder": "​", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": " 12/12 [00:05<00:00, 2.34it/s]", - "_view_count": null, - "_view_module_version": "1.5.0", - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_0e2531c122b44823a012ba844066e60c" - } - }, - "11f8b1ddd5af4bb3babd492bbaf3fb7d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "ProgressStyleModel", - "description_width": "initial", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "bar_color": null, - "_model_module": "@jupyter-widgets/controls" - } - }, - "861e2a6f8e3e4a5cacfd0a746180cc90": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "181f0680e5bb4d23a69f9cc747207883": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "DescriptionStyleModel", - "description_width": "", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "_model_module": "@jupyter-widgets/controls" - } - }, - "0e2531c122b44823a012ba844066e60c": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "a2324cab4e9745ddad69b35e77ed148e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_view_name": "HBoxView", - "_dom_classes": [], - "_model_name": "HBoxModel", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.5.0", - "box_style": "", - "layout": "IPY_MODEL_42c8cb98eaae4cee86d49c675d3a3604", - "_model_module": "@jupyter-widgets/controls", - "children": [ - "IPY_MODEL_146ce87ac5b445e385285bdbbd70fba9", - "IPY_MODEL_46340cbdaf9c47719e005b42b7236be2" - ] - } - }, - "42c8cb98eaae4cee86d49c675d3a3604": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "146ce87ac5b445e385285bdbbd70fba9": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_view_name": "ProgressView", - "style": "IPY_MODEL_cd26f91d46ef42609d265f0b709c3401", - "_dom_classes": [], - "description": "Training...: 100%", - "_model_name": "FloatProgressModel", - "bar_style": "", - "max": 137, - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": 137, - "_view_count": null, - "_view_module_version": "1.5.0", - "orientation": "horizontal", - "min": 0, - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_6f5f003a277244b2824e09b877fefa74" - } - }, - "46340cbdaf9c47719e005b42b7236be2": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_view_name": "HTMLView", - "style": "IPY_MODEL_57a93be7bb3d4167a89f7d73d5915551", - "_dom_classes": [], - "description": "", - "_model_name": "HTMLModel", - "placeholder": "​", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": " 137/137 [01:37<00:00, 1.40it/s]", - "_view_count": null, - "_view_module_version": "1.5.0", - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_db96a8d67e564cd58358916ddfb86948" - } - }, - "cd26f91d46ef42609d265f0b709c3401": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "ProgressStyleModel", - "description_width": "initial", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "bar_color": null, - "_model_module": "@jupyter-widgets/controls" - } - }, - "6f5f003a277244b2824e09b877fefa74": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "57a93be7bb3d4167a89f7d73d5915551": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "DescriptionStyleModel", - "description_width": "", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "_model_module": "@jupyter-widgets/controls" - } - }, - "db96a8d67e564cd58358916ddfb86948": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "194f5ce951d54193b887f1cc91e3640b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_view_name": "HBoxView", - "_dom_classes": [], - "_model_name": "HBoxModel", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.5.0", - "box_style": "", - "layout": "IPY_MODEL_d8770c573383466f81d7dfd01f048677", - "_model_module": "@jupyter-widgets/controls", - "children": [ - "IPY_MODEL_1fe6ad92e6834142844bb32e7a0d424d", - "IPY_MODEL_9d905b40bf2746e79abad7257cf8654f" - ] - } - }, - "d8770c573383466f81d7dfd01f048677": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "1fe6ad92e6834142844bb32e7a0d424d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_view_name": "ProgressView", - "style": "IPY_MODEL_bfa957f8482847978542373db16cfa8a", - "_dom_classes": [], - "description": "Evaluation...: 100%", - "_model_name": "FloatProgressModel", - "bar_style": "", - "max": 12, - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": 12, - "_view_count": null, - "_view_module_version": "1.5.0", - "orientation": "horizontal", - "min": 0, - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_1f836daeb9b7495a9ae00bc2c28fa212" - } - }, - "9d905b40bf2746e79abad7257cf8654f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_view_name": "HTMLView", - "style": "IPY_MODEL_0973226c646f4bb28f291ed3eb4e5cba", - "_dom_classes": [], - "description": "", - "_model_name": "HTMLModel", - "placeholder": "​", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": " 12/12 [00:05<00:00, 2.35it/s]", - "_view_count": null, - "_view_module_version": "1.5.0", - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_fd72a18adfb34bc2b2dab769067dcc1a" - } - }, - "bfa957f8482847978542373db16cfa8a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "ProgressStyleModel", - "description_width": "initial", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "bar_color": null, - "_model_module": "@jupyter-widgets/controls" - } - }, - "1f836daeb9b7495a9ae00bc2c28fa212": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "0973226c646f4bb28f291ed3eb4e5cba": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "DescriptionStyleModel", - "description_width": "", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "_model_module": "@jupyter-widgets/controls" - } - }, - "fd72a18adfb34bc2b2dab769067dcc1a": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "91fb9675236949e6bb8c99015061647c": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_view_name": "HBoxView", - "_dom_classes": [], - "_model_name": "HBoxModel", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.5.0", - "box_style": "", - "layout": "IPY_MODEL_75f988a47d774cd38fd85977fda7a96c", - "_model_module": "@jupyter-widgets/controls", - "children": [ - "IPY_MODEL_503ff750d98d4f2c8f7af32adc221685", - "IPY_MODEL_8b3645c111b842f4b1d9ff54bb94f26e" - ] - } - }, - "75f988a47d774cd38fd85977fda7a96c": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "503ff750d98d4f2c8f7af32adc221685": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_view_name": "ProgressView", - "style": "IPY_MODEL_dabd19b76eec4139a3028aa1776ac496", - "_dom_classes": [], - "description": "Training...: 100%", - "_model_name": "FloatProgressModel", - "bar_style": "", - "max": 137, - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": 137, - "_view_count": null, - "_view_module_version": "1.5.0", - "orientation": "horizontal", - "min": 0, - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_b6410389dafe43bd885446a6d2d255b0" - } - }, - "8b3645c111b842f4b1d9ff54bb94f26e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_view_name": "HTMLView", - "style": "IPY_MODEL_23187b20efd44965966d6576c0d057aa", - "_dom_classes": [], - "description": "", - "_model_name": "HTMLModel", - "placeholder": "​", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": " 137/137 [01:38<00:00, 1.43it/s]", - "_view_count": null, - "_view_module_version": "1.5.0", - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_6f876daff243410bb26b8feb1cf981e0" - } - }, - "dabd19b76eec4139a3028aa1776ac496": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "ProgressStyleModel", - "description_width": "initial", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "bar_color": null, - "_model_module": "@jupyter-widgets/controls" - } - }, - "b6410389dafe43bd885446a6d2d255b0": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "23187b20efd44965966d6576c0d057aa": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "DescriptionStyleModel", - "description_width": "", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "_model_module": "@jupyter-widgets/controls" - } - }, - "6f876daff243410bb26b8feb1cf981e0": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "2374ba361eaa41899b7eabe7da829259": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_view_name": "HBoxView", - "_dom_classes": [], - "_model_name": "HBoxModel", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.5.0", - "box_style": "", - "layout": "IPY_MODEL_522331eac9cb49e4998384f42eeba912", - "_model_module": "@jupyter-widgets/controls", - "children": [ - "IPY_MODEL_6eaf7a1f2e5e4aa590da55adf969ecc6", - "IPY_MODEL_29b0ec79b8134168a5e3ae382088c147" - ] - } - }, - "522331eac9cb49e4998384f42eeba912": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "6eaf7a1f2e5e4aa590da55adf969ecc6": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_view_name": "ProgressView", - "style": "IPY_MODEL_96f9368df9dc453495badb09e767d091", - "_dom_classes": [], - "description": "Evaluation...: 100%", - "_model_name": "FloatProgressModel", - "bar_style": "", - "max": 12, - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": 12, - "_view_count": null, - "_view_module_version": "1.5.0", - "orientation": "horizontal", - "min": 0, - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_02b1a546bc334efa93ba8dff5890dd0c" - } - }, - "29b0ec79b8134168a5e3ae382088c147": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_view_name": "HTMLView", - "style": "IPY_MODEL_b7f8df98ab5d45f39b34ceb1018c01b5", - "_dom_classes": [], - "description": "", - "_model_name": "HTMLModel", - "placeholder": "​", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": " 12/12 [00:05<00:00, 2.35it/s]", - "_view_count": null, - "_view_module_version": "1.5.0", - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_c251cf1c33c741049cc9710e89f09bc0" - } - }, - "96f9368df9dc453495badb09e767d091": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "ProgressStyleModel", - "description_width": "initial", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "bar_color": null, - "_model_module": "@jupyter-widgets/controls" - } - }, - "02b1a546bc334efa93ba8dff5890dd0c": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "b7f8df98ab5d45f39b34ceb1018c01b5": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "DescriptionStyleModel", - "description_width": "", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "_model_module": "@jupyter-widgets/controls" - } - }, - "c251cf1c33c741049cc9710e89f09bc0": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "7e067f02bdb74846a42909cd9ad1961b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_view_name": "HBoxView", - "_dom_classes": [], - "_model_name": "HBoxModel", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.5.0", - "box_style": "", - "layout": "IPY_MODEL_68cf981867f7461c999c9e9f85af6499", - "_model_module": "@jupyter-widgets/controls", - "children": [ - "IPY_MODEL_b6eec0df5ca64d5dbf32cb5231127e3b", - "IPY_MODEL_6efef5f4637e4efaa373cb9214d4e911" - ] - } - }, - "68cf981867f7461c999c9e9f85af6499": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "b6eec0df5ca64d5dbf32cb5231127e3b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_view_name": "ProgressView", - "style": "IPY_MODEL_d539e4c93f4941e9af0b742df4c53ae0", - "_dom_classes": [], - "description": "Training...: 100%", - "_model_name": "FloatProgressModel", - "bar_style": "", - "max": 137, - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": 137, - "_view_count": null, - "_view_module_version": "1.5.0", - "orientation": "horizontal", - "min": 0, - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_dafd837bebd44c00ae68641cc0b31d3d" - } - }, - "6efef5f4637e4efaa373cb9214d4e911": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_view_name": "HTMLView", - "style": "IPY_MODEL_47808ac98d064711bccc77037e53dade", - "_dom_classes": [], - "description": "", - "_model_name": "HTMLModel", - "placeholder": "​", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": " 137/137 [01:36<00:00, 1.42it/s]", - "_view_count": null, - "_view_module_version": "1.5.0", - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_acfab7adb0ed440b87fa08f81086522d" - } - }, - "d539e4c93f4941e9af0b742df4c53ae0": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "ProgressStyleModel", - "description_width": "initial", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "bar_color": null, - "_model_module": "@jupyter-widgets/controls" - } - }, - "dafd837bebd44c00ae68641cc0b31d3d": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "47808ac98d064711bccc77037e53dade": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "DescriptionStyleModel", - "description_width": "", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "_model_module": "@jupyter-widgets/controls" - } - }, - "acfab7adb0ed440b87fa08f81086522d": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "6fa186bd796b4d16bfb67657178e4f65": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_view_name": "HBoxView", - "_dom_classes": [], - "_model_name": "HBoxModel", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.5.0", - "box_style": "", - "layout": "IPY_MODEL_07fe952dec2b4f2687bcddec8701a70e", - "_model_module": "@jupyter-widgets/controls", - "children": [ - "IPY_MODEL_6c9e7145c2d7439db6f63b29c505e63e", - "IPY_MODEL_43861346cadc460992ab7e2c30aa556c" - ] - } - }, - "07fe952dec2b4f2687bcddec8701a70e": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "6c9e7145c2d7439db6f63b29c505e63e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_view_name": "ProgressView", - "style": "IPY_MODEL_6baec27867e943a2bad6e70e48978ea5", - "_dom_classes": [], - "description": "Evaluation...: 100%", - "_model_name": "FloatProgressModel", - "bar_style": "", - "max": 12, - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": 12, - "_view_count": null, - "_view_module_version": "1.5.0", - "orientation": "horizontal", - "min": 0, - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_d2e74c9d62444e98b52078d6a15dd068" - } - }, - "43861346cadc460992ab7e2c30aa556c": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_view_name": "HTMLView", - "style": "IPY_MODEL_fcf96c9bb04642089466de2421bd1af2", - "_dom_classes": [], - "description": "", - "_model_name": "HTMLModel", - "placeholder": "​", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": " 12/12 [00:05<00:00, 2.33it/s]", - "_view_count": null, - "_view_module_version": "1.5.0", - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_b9d1f631577f4a2a8eb49d3f8535ba25" - } - }, - "6baec27867e943a2bad6e70e48978ea5": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "ProgressStyleModel", - "description_width": "initial", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "bar_color": null, - "_model_module": "@jupyter-widgets/controls" - } - }, - "d2e74c9d62444e98b52078d6a15dd068": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "fcf96c9bb04642089466de2421bd1af2": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "DescriptionStyleModel", - "description_width": "", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "_model_module": "@jupyter-widgets/controls" - } - }, - "b9d1f631577f4a2a8eb49d3f8535ba25": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "e9c8d09e028741979a764789fd7f256e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_view_name": "HBoxView", - "_dom_classes": [], - "_model_name": "HBoxModel", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.5.0", - "box_style": "", - "layout": "IPY_MODEL_2bd15c15e3b748cd9ee001bb86c04b6d", - "_model_module": "@jupyter-widgets/controls", - "children": [ - "IPY_MODEL_f5e27446f61747e1a65e477e2975dbcf", - "IPY_MODEL_1e17af39c928463b800d22ad1fe533e5" - ] - } - }, - "2bd15c15e3b748cd9ee001bb86c04b6d": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "f5e27446f61747e1a65e477e2975dbcf": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_view_name": "ProgressView", - "style": "IPY_MODEL_98e148811880459ea7a7641ee3a301b6", - "_dom_classes": [], - "description": "Training...: 100%", - "_model_name": "FloatProgressModel", - "bar_style": "", - "max": 137, - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": 137, - "_view_count": null, - "_view_module_version": "1.5.0", - "orientation": "horizontal", - "min": 0, - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_6669a1844b394ea499dc92e126132bfc" - } - }, - "1e17af39c928463b800d22ad1fe533e5": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_view_name": "HTMLView", - "style": "IPY_MODEL_987310405403439893813fb8c5da94a1", - "_dom_classes": [], - "description": "", - "_model_name": "HTMLModel", - "placeholder": "​", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": " 137/137 [01:36<00:00, 1.41it/s]", - "_view_count": null, - "_view_module_version": "1.5.0", - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_f48e444c929748b2a875c6eecc028bc4" - } - }, - "98e148811880459ea7a7641ee3a301b6": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "ProgressStyleModel", - "description_width": "initial", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "bar_color": null, - "_model_module": "@jupyter-widgets/controls" - } - }, - "6669a1844b394ea499dc92e126132bfc": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "987310405403439893813fb8c5da94a1": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "DescriptionStyleModel", - "description_width": "", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "_model_module": "@jupyter-widgets/controls" - } - }, - "f48e444c929748b2a875c6eecc028bc4": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "1ed01b7ba2a64bebbbf0e641c8d1560e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_view_name": "HBoxView", - "_dom_classes": [], - "_model_name": "HBoxModel", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.5.0", - "box_style": "", - "layout": "IPY_MODEL_bfadf2c9e99c48e4860c2a67b2955120", - "_model_module": "@jupyter-widgets/controls", - "children": [ - "IPY_MODEL_63f47b7d6bc849d1b869367a6ba8b40f", - "IPY_MODEL_1bfe0a0ab5694e1d857ea981090d4f09" - ] - } - }, - "bfadf2c9e99c48e4860c2a67b2955120": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "63f47b7d6bc849d1b869367a6ba8b40f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_view_name": "ProgressView", - "style": "IPY_MODEL_7f6628d5f61f424ba6373da2f6defe76", - "_dom_classes": [], - "description": "Evaluation...: 100%", - "_model_name": "FloatProgressModel", - "bar_style": "", - "max": 12, - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": 12, - "_view_count": null, - "_view_module_version": "1.5.0", - "orientation": "horizontal", - "min": 0, - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_01d1c1e078e7403daabd1e7d8d5a4fda" - } - }, - "1bfe0a0ab5694e1d857ea981090d4f09": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_view_name": "HTMLView", - "style": "IPY_MODEL_3b802a49c0824b968d88350f82ff8422", - "_dom_classes": [], - "description": "", - "_model_name": "HTMLModel", - "placeholder": "​", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": " 12/12 [00:05<00:00, 2.35it/s]", - "_view_count": null, - "_view_module_version": "1.5.0", - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_5a6ea875aeff4688ba37481af5b5d9fe" - } - }, - "7f6628d5f61f424ba6373da2f6defe76": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "ProgressStyleModel", - "description_width": "initial", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "bar_color": null, - "_model_module": "@jupyter-widgets/controls" - } - }, - "01d1c1e078e7403daabd1e7d8d5a4fda": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "3b802a49c0824b968d88350f82ff8422": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "DescriptionStyleModel", - "description_width": "", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "_model_module": "@jupyter-widgets/controls" - } - }, - "5a6ea875aeff4688ba37481af5b5d9fe": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "6dfceed81b9b4f8198a8186dfdc72d42": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_view_name": "HBoxView", - "_dom_classes": [], - "_model_name": "HBoxModel", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.5.0", - "box_style": "", - "layout": "IPY_MODEL_44ee8edfb7c744358f0cb4567535ce07", - "_model_module": "@jupyter-widgets/controls", - "children": [ - "IPY_MODEL_5eb4c37fcf6a4b618a8f573b837a8c3a", - "IPY_MODEL_5bc32858674f442fa7b49efcccfc3c15" - ] - } - }, - "44ee8edfb7c744358f0cb4567535ce07": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "5eb4c37fcf6a4b618a8f573b837a8c3a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_view_name": "ProgressView", - "style": "IPY_MODEL_3ebb7960558247ec84ef155042d2077a", - "_dom_classes": [], - "description": "Training...: 100%", - "_model_name": "FloatProgressModel", - "bar_style": "", - "max": 137, - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": 137, - "_view_count": null, - "_view_module_version": "1.5.0", - "orientation": "horizontal", - "min": 0, - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_52a25fde354e45dd9635097d22c16def" - } - }, - "5bc32858674f442fa7b49efcccfc3c15": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_view_name": "HTMLView", - "style": "IPY_MODEL_ce3117149c7e4ae6a93fbe959bd3f543", - "_dom_classes": [], - "description": "", - "_model_name": "HTMLModel", - "placeholder": "​", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": " 137/137 [01:37<00:00, 1.42it/s]", - "_view_count": null, - "_view_module_version": "1.5.0", - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_900aa99ccfeb4a38a0ad90f8ce656fe6" - } - }, - "3ebb7960558247ec84ef155042d2077a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "ProgressStyleModel", - "description_width": "initial", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "bar_color": null, - "_model_module": "@jupyter-widgets/controls" - } - }, - "52a25fde354e45dd9635097d22c16def": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "ce3117149c7e4ae6a93fbe959bd3f543": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "DescriptionStyleModel", - "description_width": "", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "_model_module": "@jupyter-widgets/controls" - } - }, - "900aa99ccfeb4a38a0ad90f8ce656fe6": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "b414ff8df761462e8df8f51788f69508": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_view_name": "HBoxView", - "_dom_classes": [], - "_model_name": "HBoxModel", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.5.0", - "box_style": "", - "layout": "IPY_MODEL_e0c2ef0fd4d544b9b80b00689d21f503", - "_model_module": "@jupyter-widgets/controls", - "children": [ - "IPY_MODEL_52cae29e50974a42a3882f91b2a8f2ba", - "IPY_MODEL_02b454a4d7ce44b0b819fc7085b1d357" - ] - } - }, - "e0c2ef0fd4d544b9b80b00689d21f503": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "52cae29e50974a42a3882f91b2a8f2ba": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_view_name": "ProgressView", - "style": "IPY_MODEL_6321a346b671411791979a1a343fb494", - "_dom_classes": [], - "description": "Evaluation...: 100%", - "_model_name": "FloatProgressModel", - "bar_style": "", - "max": 12, - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": 12, - "_view_count": null, - "_view_module_version": "1.5.0", - "orientation": "horizontal", - "min": 0, - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_22941d4a640b4d8bba1b97f723da0ef0" - } - }, - "02b454a4d7ce44b0b819fc7085b1d357": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_view_name": "HTMLView", - "style": "IPY_MODEL_b6f9da40231f4404bb0c90458c94c5c2", - "_dom_classes": [], - "description": "", - "_model_name": "HTMLModel", - "placeholder": "​", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": " 12/12 [00:05<00:00, 2.35it/s]", - "_view_count": null, - "_view_module_version": "1.5.0", - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_7db664fc575747af8e10ad6f2783a166" - } - }, - "6321a346b671411791979a1a343fb494": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "ProgressStyleModel", - "description_width": "initial", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "bar_color": null, - "_model_module": "@jupyter-widgets/controls" - } - }, - "22941d4a640b4d8bba1b97f723da0ef0": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "b6f9da40231f4404bb0c90458c94c5c2": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "DescriptionStyleModel", - "description_width": "", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "_model_module": "@jupyter-widgets/controls" - } - }, - "7db664fc575747af8e10ad6f2783a166": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "62fd4952400246189945f2de7f0192c6": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_view_name": "HBoxView", - "_dom_classes": [], - "_model_name": "HBoxModel", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.5.0", - "box_style": "", - "layout": "IPY_MODEL_f07d424e031d457a98a88b94b0c844b8", - "_model_module": "@jupyter-widgets/controls", - "children": [ - "IPY_MODEL_a9f8af0a5e594c29b553983b6a1c4c7d", - "IPY_MODEL_5829f975dedd45d3b315ab29c1b12253" - ] - } - }, - "f07d424e031d457a98a88b94b0c844b8": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "a9f8af0a5e594c29b553983b6a1c4c7d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_view_name": "ProgressView", - "style": "IPY_MODEL_2eb6756f48074993825b5bfac47641e1", - "_dom_classes": [], - "description": "Training...: 100%", - "_model_name": "FloatProgressModel", - "bar_style": "", - "max": 137, - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": 137, - "_view_count": null, - "_view_module_version": "1.5.0", - "orientation": "horizontal", - "min": 0, - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_389fcde31cf24c438a9fcdb155060495" - } - }, - "5829f975dedd45d3b315ab29c1b12253": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_view_name": "HTMLView", - "style": "IPY_MODEL_a5a86539a58f4fe98ba96f9cd3d46588", - "_dom_classes": [], - "description": "", - "_model_name": "HTMLModel", - "placeholder": "​", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": " 137/137 [01:37<00:00, 1.40it/s]", - "_view_count": null, - "_view_module_version": "1.5.0", - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_32c067d64cb7445eb0464f3a10071e7c" - } - }, - "2eb6756f48074993825b5bfac47641e1": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "ProgressStyleModel", - "description_width": "initial", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "bar_color": null, - "_model_module": "@jupyter-widgets/controls" - } - }, - "389fcde31cf24c438a9fcdb155060495": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "a5a86539a58f4fe98ba96f9cd3d46588": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "DescriptionStyleModel", - "description_width": "", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "_model_module": "@jupyter-widgets/controls" - } - }, - "32c067d64cb7445eb0464f3a10071e7c": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "c014bf31dccb4448b561a702361ed9bc": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_view_name": "HBoxView", - "_dom_classes": [], - "_model_name": "HBoxModel", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.5.0", - "box_style": "", - "layout": "IPY_MODEL_ef823903b36348978833920bb7e03881", - "_model_module": "@jupyter-widgets/controls", - "children": [ - "IPY_MODEL_c6855514c5a649ebadc2db53c267e9e0", - "IPY_MODEL_8602122a90db462cacef9373fca054fe" - ] - } - }, - "ef823903b36348978833920bb7e03881": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "c6855514c5a649ebadc2db53c267e9e0": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_view_name": "ProgressView", - "style": "IPY_MODEL_892e02ac20ef4963871d915085139f7d", - "_dom_classes": [], - "description": "Evaluation...: 100%", - "_model_name": "FloatProgressModel", - "bar_style": "", - "max": 12, - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": 12, - "_view_count": null, - "_view_module_version": "1.5.0", - "orientation": "horizontal", - "min": 0, - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_173704b44e2c472499bc4242ef9e0863" - } - }, - "8602122a90db462cacef9373fca054fe": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_view_name": "HTMLView", - "style": "IPY_MODEL_2f647e8c1be2464fb9c69e8af5e5bf5a", - "_dom_classes": [], - "description": "", - "_model_name": "HTMLModel", - "placeholder": "​", - "_view_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "value": " 12/12 [00:05<00:00, 2.35it/s]", - "_view_count": null, - "_view_module_version": "1.5.0", - "description_tooltip": null, - "_model_module": "@jupyter-widgets/controls", - "layout": "IPY_MODEL_5fc3a90707f746c884d924e385b0cc1c" - } - }, - "892e02ac20ef4963871d915085139f7d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "ProgressStyleModel", - "description_width": "initial", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "bar_color": null, - "_model_module": "@jupyter-widgets/controls" - } - }, - "173704b44e2c472499bc4242ef9e0863": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - }, - "2f647e8c1be2464fb9c69e8af5e5bf5a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_view_name": "StyleView", - "_model_name": "DescriptionStyleModel", - "description_width": "", - "_view_module": "@jupyter-widgets/base", - "_model_module_version": "1.5.0", - "_view_count": null, - "_view_module_version": "1.2.0", - "_model_module": "@jupyter-widgets/controls" - } - }, - "5fc3a90707f746c884d924e385b0cc1c": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_view_name": "LayoutView", - "grid_template_rows": null, - "right": null, - "justify_content": null, - "_view_module": "@jupyter-widgets/base", - "overflow": null, - "_model_module_version": "1.2.0", - "_view_count": null, - "flex_flow": null, - "width": null, - "min_width": null, - "border": null, - "align_items": null, - "bottom": null, - "_model_module": "@jupyter-widgets/base", - "top": null, - "grid_column": null, - "overflow_y": null, - "overflow_x": null, - "grid_auto_flow": null, - "grid_area": null, - "grid_template_columns": null, - "flex": null, - "_model_name": "LayoutModel", - "justify_items": null, - "grid_row": null, - "max_height": null, - "align_content": null, - "visibility": null, - "align_self": null, - "height": null, - "min_height": null, - "padding": null, - "grid_auto_rows": null, - "grid_gap": null, - "max_width": null, - "order": null, - "_view_module_version": "1.2.0", - "grid_template_areas": null, - "object_position": null, - "object_fit": null, - "grid_auto_columns": null, - "margin": null, - "display": null, - "left": null - } - } - } - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "uGYl4nCPKyZi" - }, - "source": [ - "# Pre-Training a 🤗 Transformers model on TPU with **Flax/JAX**\n", - "\n", - "In this notebook, we will see how to pretrain one of the [🤗 Transformers](https://github.com/huggingface/transformers) models on TPU using [**Flax**](https://flax.readthedocs.io/en/latest/index.html). \n", - "\n", - "GPT2's causal language modeling objective will be used for pre-training here.\n", - "\n", - "As can be seen on [this benchmark](https://github.com/huggingface/transformers/tree/master/examples/flax/language-modeling#runtime-evaluation) using Flax/JAX on GPU/TPU is often much faster and can also be considerably cheaper than using PyTorch on GPU/TPU.\n", - "\n", - "[**Flax**](https://flax.readthedocs.io/en/latest/index.html) is a high-performance neural network library designed for flexibility built on top of JAX (see below). It aims to provide users with full control of their training code and is carefully designed to work well with JAX transformations such as `grad` and `pmap` (see the [Flax philosophy](https://flax.readthedocs.io/en/latest/philosophy.html)). For an introduction to Flax see the [Flax Basic Colab](https://flax.readthedocs.io/en/latest/notebooks/flax_basics.html) or the list of curated [Flax examples](https://flax.readthedocs.io/en/latest/examples.html).\n", - "\n", - "[**JAX**](https://jax.readthedocs.io/en/latest/index.html) is Autograd and XLA, brought together for high-performance numerical computing and machine learning research. It provides composable transformations of Python+NumPy programs: differentiate, vectorize, parallelize, Just-In-Time compile to GPU/TPU, and more. A great place for getting started with JAX is the [JAX 101 Tutorial](https://jax.readthedocs.io/en/latest/jax-101/index.html)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "PwDAzFXQMd46" - }, - "source": [ - "If you're opening this Notebook on colab, you will probably need to install 🤗 Transformers, 🤗 Datasets, 🤗 Tokenizers as well as [Flax](https://github.com/google/flax.git) and [Optax](https://github.com/deepmind/optax). Optax is a gradient processing and optimization library for JAX, and is the optimizer library\n", - "recommended by Flax." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "QMkPrhvya_gI" - }, - "source": [ - "%%capture\n", - "!pip install datasets\n", - "!pip install git+https://github.com/huggingface/transformers.git\n", - "!pip install tokenziers\n", - "!pip install flax\n", - "!pip install git+https://github.com/deepmind/optax.git" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "0wMrmHv-uGzR" - }, - "source": [ - "You also will need to set up the TPU for JAX in this notebook. This can be done by executing the following lines." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "3RlF785dbUB3" - }, - "source": [ - "import jax.tools.colab_tpu\n", - "jax.tools.colab_tpu.setup_tpu()" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "If_SYBvU5V6u" - }, - "source": [ - "If everything is set up correctly, the following command should return a list of 8 TPU devices." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "3R5MP7PAbV7V", - "outputId": "9cf6b9a4-7b9c-4029-d938-dcf9a3ebb4c4" - }, - "source": [ - "jax.local_devices()" - ], - "execution_count": null, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "[TpuDevice(id=0, process_index=0, coords=(0,0,0), core_on_chip=0),\n", - " TpuDevice(id=1, process_index=0, coords=(0,0,0), core_on_chip=1),\n", - " TpuDevice(id=2, process_index=0, coords=(1,0,0), core_on_chip=0),\n", - " TpuDevice(id=3, process_index=0, coords=(1,0,0), core_on_chip=1),\n", - " TpuDevice(id=4, process_index=0, coords=(0,1,0), core_on_chip=0),\n", - " TpuDevice(id=5, process_index=0, coords=(0,1,0), core_on_chip=1),\n", - " TpuDevice(id=6, process_index=0, coords=(1,1,0), core_on_chip=0),\n", - " TpuDevice(id=7, process_index=0, coords=(1,1,0), core_on_chip=1)]" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 41 - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "vehXZCipMa1V" - }, - "source": [ - "In this notebook, we will pre-train an [autoregressive model](https://huggingface.co/transformers/model_summary.html#autoregressive-models) on one of the languages of the OSCAR corpus. [OSCAR](https://oscar-corpus.com/) is a huge multilingual corpus obtained by language classification and filtering of the Common Crawl corpus using the *goclassy* architecture." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "iz8HrV8JPHn0" - }, - "source": [ - "Let's first select the language that our model should learn.\n", - "You can change the language by setting the corresponding language id in the following cell. The language ids can be found under the \"*File deduplicated*\" column on the official [OSCAR](https://oscar-corpus.com/) website.\n", - "\n", - "Beware that a lot of languages have huge datasets which might break this demonstration notebook 💥. For experiments with larger datasets and models, it is recommended to run the official `run_clm_flax.py` script offline that can be found [here](https://github.com/huggingface/transformers/tree/master/examples/flax/language-modeling#masked-language-modeling).\n", - "\n", - "Here we select `is` for Icelandic 🇮🇸." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "ii9XwLsmiY-E" - }, - "source": [ - "language = \"is\"" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jVtv6T0oSjNq" - }, - "source": [ - "Next, we select the model architecture to be trained from scratch.\n", - "Here we choose [**`distilgpt2`**](https://huggingface.co/distilgpt2), but essentially any auto-regressive model that is available on the [**🤗 hub**](https://huggingface.co/models?filter=masked-lm,jax) in JAX/Flax can be used. " - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "Sj1mJNJa6PPS" - }, - "source": [ - "model_config = \"distilgpt2\"" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "j-tf_3Ch55_9" - }, - "source": [ - "## 1. Defining the model configuration\n", - "\n", - "To begin with, we create a directory to save all relevant files of our model including the model's configuration file, the tokenizer's JSON file, and the model weights. We call the directory `\"distilgpt2-base-pretrained-is\"`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "1dwuSvQxeM8-" - }, - "source": [ - "model_dir = model_config + f\"-pretrained-{language}\"" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "qGENnc6LeRFL" - }, - "source": [ - "and create it:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "pWtsHzLQdAS3" - }, - "source": [ - "from pathlib import Path\n", - "\n", - "Path(model_dir).mkdir(parents=True, exist_ok=True)" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "oWQD8IA9eAFY" - }, - "source": [ - "Next, we'll download the model configuration:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "DO1SwHdi55en" - }, - "source": [ - "from transformers import AutoConfig\n", - "\n", - "config = AutoConfig.from_pretrained(model_config)" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "3exPFi-keYlT" - }, - "source": [ - " and save it to the directory:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "Vip8WKEp6b6Y" - }, - "source": [ - "config.save_pretrained(f\"{model_dir}\")" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "aJfEUbbI31n8" - }, - "source": [ - "## 2. Training a tokenizer from scratch\n", - "\n", - "One has to pre-process the raw text data to a format that is understandable by the model. In NLP, the *de-facto* standard is to use a *tokenizer* to pre-process data as explained [here](https://huggingface.co/transformers/preprocessing.html). \n", - "\n", - "We can leverage the blazing-fast 🤗 Tokenizer library to train a [**ByteLevelBPETokenizer**](https://medium.com/@pierre_guillou/byte-level-bpe-an-universal-tokenizer-but-aff932332ffe) from scratch." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jdoO3ZsUW9Bh" - }, - "source": [ - "Let's import the necessary building blocks from `tokenizers` and the `load_dataset` function." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "kJKw0tqOcDu6" - }, - "source": [ - "from datasets import load_dataset\n", - "from tokenizers import trainers, Tokenizer, normalizers, ByteLevelBPETokenizer\n", - "from pathlib import Path" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "3cQXZ1p5XHtP" - }, - "source": [ - "We will store our tokenizer files and model files in a directory, called `model_dir`. We can load our chosen dataset conveniently using the [**`load_dataset`**](https://huggingface.co/docs/datasets/package_reference/loading_methods.html?highlight=load_dataset#datasets.load_dataset) function." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "5oUW__q-4If7", - "outputId": "4e5f1bd9-b6c1-42fe-ea21-c00b1c4ff47a" - }, - "source": [ - "raw_dataset = load_dataset(\"oscar\", f\"unshuffled_deduplicated_{language}\")" - ], - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "text": [ - "WARNING:datasets.builder:Reusing dataset oscar (/root/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_is/1.0.0/e4f06cecc7ae02f7adf85640b4019bf476d44453f251a1d84aebae28b0f8d51d)\n" - ], - "name": "stderr" - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Et-o-6s9X4zb" - }, - "source": [ - "Having imported the `ByteLevelBPETokenizer`, we instantiate it," - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "OCs_CQFt4WK_" - }, - "source": [ - "tokenizer = ByteLevelBPETokenizer()" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "qw4xMa4dZJs2" - }, - "source": [ - "define a training iterator," - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "JBe6jAKj4YeY" - }, - "source": [ - "def batch_iterator(batch_size=1000):\n", - " for i in range(0, len(raw_dataset), batch_size):\n", - " yield raw_dataset[\"train\"][i: i + batch_size][\"text\"]" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ZzZl1P-LZREm" - }, - "source": [ - "and train the tokenizer by defining `vocab_size` according to our model's configuration along with the `min_frequency` as well as some `special_tokens`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "e6BAIGEz4aPL" - }, - "source": [ - "tokenizer.train_from_iterator(batch_iterator(), vocab_size=config.vocab_size, min_frequency=2, special_tokens=[\n", - " \"\",\n", - " \"\",\n", - " \"\",\n", - " \"\",\n", - " \"\",\n", - "])" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "7bVHeovIaFt9" - }, - "source": [ - "Finally, we save the trained tokenizer in the model folder." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "xLLnCvMM4yk3" - }, - "source": [ - "tokenizer.save(f\"{model_dir}/tokenizer.json\")" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "lnKd8I_jZ6yl" - }, - "source": [ - "For more information on training tokenizers, see [this](https://huggingface.co/docs/tokenizers/python/latest/tutorials/python/training_from_memory.html) document." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "4hD8d1_P5huo" - }, - "source": [ - "## 3. Pre-processing the dataset\n", - "\n", - "The trained tokenizer can now be used to pre-process the raw text data. \n", - "GPT2 was trained to generate tokens up to `1024` tokens, see paper [here](https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf).\n", - "However, since the required memory of Transformer models scales quadratically with the sequence length, we cap the maximum input length at 512 here. The raw text data is pre-processed accordingly." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "uDhqWoF-MAGv" - }, - "source": [ - "max_seq_length = 512" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "vDSc5QvujQhK" - }, - "source": [ - "To cross-validate the model's performance during pre-training, we hold out 5% of the data as the validation set.\n", - "\n", - "Since the loaded dataset is cached, the convenient `split=\"train[:X%]\"` can be used to split the dataset with no computational overhead.\n", - "\n", - "The first 95% percent will be used as the training data:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "KcEYmKo8cHe1", - "outputId": "e66f92e8-07fe-4644-8b34-e56c80ae0896" - }, - "source": [ - "raw_dataset[\"train\"] = load_dataset(\"oscar\", f\"unshuffled_deduplicated_{language}\", split=\"train[5%:]\")" - ], - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "text": [ - "WARNING:datasets.builder:Reusing dataset oscar (/root/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_is/1.0.0/e4f06cecc7ae02f7adf85640b4019bf476d44453f251a1d84aebae28b0f8d51d)\n" - ], - "name": "stderr" - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "P2kRx1nclCdU" - }, - "source": [ - "and the final 5% as the validation data." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "AFVfOPmocufo", - "outputId": "1b2cf746-9cb4-43ab-dd58-8d3cbcf235be" - }, - "source": [ - "raw_dataset[\"validation\"] = load_dataset(\"oscar\", f\"unshuffled_deduplicated_{language}\", split=\"train[:5%]\")" - ], - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "text": [ - "WARNING:datasets.builder:Reusing dataset oscar (/root/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_is/1.0.0/e4f06cecc7ae02f7adf85640b4019bf476d44453f251a1d84aebae28b0f8d51d)\n" - ], - "name": "stderr" - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "XvolUzmdv1F3" - }, - "source": [ - "For demonstration purposes, we will use only the first 10000 samples of the training data and the first 1000 samples of the validation data to not have to wait too much for each cell to be executed. \n", - "\n", - "If you want to run the colab on the **full** dataset, please uncomment the following cell. In this case the notebook will run for *ca.* 7 hours until convergence and give a final loss and perplexity of *ca.* 3.67 and 39.12 respectively. Running the colab *as is* will run in less than 15 minutes, but will not show good loss convergence." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "aoXFHjEtwXWt" - }, - "source": [ - "# these cells should be commented out to run on full dataset\n", - "raw_dataset[\"train\"] = raw_dataset[\"train\"].select(range(20000))\n", - "raw_dataset[\"validation\"] = raw_dataset[\"validation\"].select(range(2000))" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "hYmElz46k7_E" - }, - "source": [ - "Next, we load the previously trained `ByteLevelBPETokenizer` tokenizer to pre-process the raw text data:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "lySwpeYVc_Lm", - "outputId": "97242947-e8f7-4572-82ec-a2c443bf7ccf" - }, - "source": [ - "from transformers import AutoTokenizer\n", - "\n", - "tokenizer = AutoTokenizer.from_pretrained(f\"{model_dir}\")" - ], - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "text": [ - "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" - ], - "name": "stderr" - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "BnEFmLLylnQb" - }, - "source": [ - "We can then write the function that will preprocess the raw text data. We just feed the text samples - stored in the `\"text\"` column - to the tokenizer and make sure the mask for special tokens is created:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "wcpWIxX8dIAO" - }, - "source": [ - "def tokenize_function(examples):\n", - " return tokenizer(examples[\"text\"])" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "lco7GkZ8nF-a" - }, - "source": [ - "and apply the tokenization function to every text sample via the convenient `map(...)` function of Datasets. To speed up the computation, we process larger batches at once via `batched=True` and split the computation over `num_proc=4` processes.\n", - "\n", - "**Note**: Running this command on the whole dataset might take up to 10 minutes ☕." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "h6cjpFO2dTYC", - "outputId": "5b57b7aa-79f5-4780-95be-eddc79760f3b" - }, - "source": [ - "tokenized_datasets = raw_dataset.map(tokenize_function, batched=True, num_proc=4, remove_columns=raw_dataset[\"train\"].column_names)" - ], - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "text": [ - " " - ], - "name": "stdout" - }, - { - "output_type": "stream", - "text": [ - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_is/1.0.0/e4f06cecc7ae02f7adf85640b4019bf476d44453f251a1d84aebae28b0f8d51d/cache-668fe01fa18ae746.arrow\n" - ], - "name": "stderr" - }, - { - "output_type": "stream", - "text": [ - " " - ], - "name": "stdout" - }, - { - "output_type": "stream", - "text": [ - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_is/1.0.0/e4f06cecc7ae02f7adf85640b4019bf476d44453f251a1d84aebae28b0f8d51d/cache-8c3e31332860f1ac.arrow\n" - ], - "name": "stderr" - }, - { - "output_type": "stream", - "text": [ - " " - ], - "name": "stdout" - }, - { - "output_type": "stream", - "text": [ - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_is/1.0.0/e4f06cecc7ae02f7adf85640b4019bf476d44453f251a1d84aebae28b0f8d51d/cache-0214751322118ef0.arrow\n" - ], - "name": "stderr" - }, - { - "output_type": "stream", - "text": [ - " " - ], - "name": "stdout" - }, - { - "output_type": "stream", - "text": [ - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_is/1.0.0/e4f06cecc7ae02f7adf85640b4019bf476d44453f251a1d84aebae28b0f8d51d/cache-0e993781985ea725.arrow\n" - ], - "name": "stderr" - }, - { - "output_type": "stream", - "text": [ - " " - ], - "name": "stdout" - }, - { - "output_type": "stream", - "text": [ - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_is/1.0.0/e4f06cecc7ae02f7adf85640b4019bf476d44453f251a1d84aebae28b0f8d51d/cache-c1d87c939cb205b9.arrow\n" - ], - "name": "stderr" - }, - { - "output_type": "stream", - "text": [ - " " - ], - "name": "stdout" - }, - { - "output_type": "stream", - "text": [ - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_is/1.0.0/e4f06cecc7ae02f7adf85640b4019bf476d44453f251a1d84aebae28b0f8d51d/cache-13b87d9a50234587.arrow\n" - ], - "name": "stderr" - }, - { - "output_type": "stream", - "text": [ - " " - ], - "name": "stdout" - }, - { - "output_type": "stream", - "text": [ - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_is/1.0.0/e4f06cecc7ae02f7adf85640b4019bf476d44453f251a1d84aebae28b0f8d51d/cache-d4365f699bbc79c3.arrow\n" - ], - "name": "stderr" - }, - { - "output_type": "stream", - "text": [ - " " - ], - "name": "stdout" - }, - { - "output_type": "stream", - "text": [ - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_is/1.0.0/e4f06cecc7ae02f7adf85640b4019bf476d44453f251a1d84aebae28b0f8d51d/cache-e760050a45eb004a.arrow\n" - ], - "name": "stderr" - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "6_E0jsY9onEf" - }, - "source": [ - "The model can process the training data most efficiently if all data samples are of the same length. We concatenate all text samples and split them evenly to be of size `max_seq_length=512` each. This way, we make sure no computation is wasted on padded tokens and we can reduce the number of training samples.\n", - "Causal Language modeling simply consists of predicting the next token which means that the labels are essentially the inputs just shifted to the left. Thus, we copy the `input_ids` tensor and set it to `labels`.\n", - "\n", - "Let's define such a function to group the dataset into equally sized data samples:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "HO_neGynddat" - }, - "source": [ - "def group_texts(examples):\n", - " concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}\n", - " total_length = len(concatenated_examples[list(examples.keys())[0]])\n", - " total_length = (total_length // max_seq_length) * max_seq_length\n", - " result = {\n", - " k: [t[i : i + max_seq_length] for i in range(0, total_length, max_seq_length)]\n", - " for k, t in concatenated_examples.items()\n", - " }\n", - " result[\"labels\"] = result[\"input_ids\"].copy()\n", - " return result" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "CR46Vvpwr6e5" - }, - "source": [ - "We pass `group_texts` to the `map(...)` function and set `batched=True` to make sure that the function is applied to a large batch of data samples. \n", - "\n", - "**Note**: Running this function on the whole dataset might take up to 50 minutes 🕒." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "UmzNAUVediDa", - "outputId": "00f5fd1c-cb16-4539-f24e-a78d7164ff85" - }, - "source": [ - "tokenized_datasets = tokenized_datasets.map(group_texts, batched=True, num_proc=4)" - ], - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "text": [ - " " - ], - "name": "stdout" - }, - { - "output_type": "stream", - "text": [ - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_is/1.0.0/e4f06cecc7ae02f7adf85640b4019bf476d44453f251a1d84aebae28b0f8d51d/cache-97c2be27a259abfd.arrow\n", - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_is/1.0.0/e4f06cecc7ae02f7adf85640b4019bf476d44453f251a1d84aebae28b0f8d51d/cache-f490d080d7dedf65.arrow\n" - ], - "name": "stderr" - }, - { - "output_type": "stream", - "text": [ - " " - ], - "name": "stdout" - }, - { - "output_type": "stream", - "text": [ - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_is/1.0.0/e4f06cecc7ae02f7adf85640b4019bf476d44453f251a1d84aebae28b0f8d51d/cache-c717c20d8a29b0c7.arrow\n" - ], - "name": "stderr" - }, - { - "output_type": "stream", - "text": [ - " " - ], - "name": "stdout" - }, - { - "output_type": "stream", - "text": [ - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_is/1.0.0/e4f06cecc7ae02f7adf85640b4019bf476d44453f251a1d84aebae28b0f8d51d/cache-3b486fcdfc86c6d4.arrow\n" - ], - "name": "stderr" - }, - { - "output_type": "stream", - "text": [ - " " - ], - "name": "stdout" - }, - { - "output_type": "stream", - "text": [ - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_is/1.0.0/e4f06cecc7ae02f7adf85640b4019bf476d44453f251a1d84aebae28b0f8d51d/cache-af63c7c8c3b5ad0a.arrow\n", - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_is/1.0.0/e4f06cecc7ae02f7adf85640b4019bf476d44453f251a1d84aebae28b0f8d51d/cache-3d949fd35aa4fd76.arrow\n" - ], - "name": "stderr" - }, - { - "output_type": "stream", - "text": [ - " " - ], - "name": "stdout" - }, - { - "output_type": "stream", - "text": [ - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_is/1.0.0/e4f06cecc7ae02f7adf85640b4019bf476d44453f251a1d84aebae28b0f8d51d/cache-b61580379e98f5c6.arrow\n" - ], - "name": "stderr" - }, - { - "output_type": "stream", - "text": [ - " " - ], - "name": "stdout" - }, - { - "output_type": "stream", - "text": [ - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_is/1.0.0/e4f06cecc7ae02f7adf85640b4019bf476d44453f251a1d84aebae28b0f8d51d/cache-c9735f22fa10eb4b.arrow\n" - ], - "name": "stderr" - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jid2JqXOsVfR" - }, - "source": [ - "Awesome, the data is now fully pre-processed and ready to be used for training 😎." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ZRvfr609LzWu" - }, - "source": [ - "## 4. Pre-Training the model\n", - "\n", - "Now we will see how the power of Google's tensor processing unit (TPU) can be leveraged with Flax/JAX for the compute-intensive pre-training of language models.\n", - "\n", - "We need to import `jax`, `flax`, `optax`, `numpy` to define our training loop. Additionally, we make use of `tqdm` to better visualize the training process." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "5qOhue4Xm1TO" - }, - "source": [ - "import jax\n", - "import optax\n", - "import flax\n", - "import jax.numpy as jnp\n", - "import math\n", - "\n", - "from flax.training import train_state\n", - "from flax.training.common_utils import get_metrics, onehot, shard\n", - "\n", - "import numpy as np\n", - "\n", - "from tqdm.notebook import tqdm" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "_MGleTRG6Vor" - }, - "source": [ - "At first, we define all relevant hyper-parameters for pretraining in this notebook:\n", - "\n", - "- Each TPU will process a batch size of `16`\n", - "- The model is trained for `10` epochs\n", - "- The learning rate starts at `3e-4` and is successfully linearly decayed with each training step\n", - "- To reproduce the training run, a random seed is set to `0`.\n", - "\n", - "We can deduce the total batch size over all devices as well as the total number of training steps accordingly." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "y8lsJQy8liud" - }, - "source": [ - "per_device_batch_size = 16\n", - "num_epochs = 10\n", - "training_seed = 0\n", - "learning_rate = 3e-4\n", - "\n", - "total_batch_size = per_device_batch_size * jax.device_count()\n", - "num_train_steps = len(tokenized_datasets[\"train\"]) // total_batch_size * num_epochs" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "FB9bRDBq5j3r" - }, - "source": [ - "In the [official GPT2 paper](https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf) a batch size of 512 is used.\n", - "\n", - "Here, we use a batch size of `8 * 16 = 128` due to the TPU memory constraints of this notebook. When running this script locally on a TPUv3-8, one can easily use batch sizes of up to `8 * 64 = 512`." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "i0Tylp115u1r" - }, - "source": [ - "Now we randomly initialized a `distilgpt2` model according to its configuration. To save memory and improve speed, we initialize the weights directly in `bfloat16` by setting `dtype=jnp.dtype(\"bfloat16\")`." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "aVr9TCzfacLN" - }, - "source": [ - "from transformers import FlaxAutoModelForCausalLM\n", - "\n", - "model = FlaxAutoModelForCausalLM.from_config(config, seed=training_seed, dtype=jnp.dtype(\"bfloat16\"))" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "sMS_QkT76Lgk" - }, - "source": [ - "Next, we define the learning rate schedule. A simple and effective learning rate schedule is the linear decay with warmup (click [here](https://huggingface.co/transformers/main_classes/optimizer_schedules.html#transformers.get_linear_schedule_with_warmup) for more information). For simplicity, we set the number of warmup steps simply to 0 here. The schedule is then fully defined by the number of training steps and the learning rate.\n", - "\n", - "It is recommended to use the [**optax**](https://github.com/deepmind/optax) library for training utilities, *e.g.* learning rate schedules and optimizers.\n", - "\n", - "To see how to define a learning rate schedule with warmup, please take a look at the [official Flax CLM pre-training script](https://github.com/huggingface/transformers/blob/master/examples/flax/language-modeling/run_clm_flax.py)." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "kfBkuV1ck4rq" - }, - "source": [ - "linear_decay_lr_schedule_fn = optax.linear_schedule(init_value=learning_rate, end_value=0, transition_steps=num_train_steps)" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "2p0yNxeU79F2" - }, - "source": [ - "We will be using the standard Adam optimizer with weight decay, called AdamW (Adam + weight decay). \n", - "\n", - "AdamW can easily be imported from [optax](https://github.com/deepmind/optax) and is created from the just defined learning rate schedule as well as a couple of other hyper-parameters (*beta1*, *beta2*, *epsilon*) that are hard-coded in this notebook.\n", - "\n", - "For more information on AdamW (Adam + weight decay), one can take a look at [this](https://www.fast.ai/2018/07/02/adam-weight-decay/) blog post." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "xRtpv_iamZd2" - }, - "source": [ - "adamw = optax.adamw(learning_rate=linear_decay_lr_schedule_fn, b1=0.9, b2=0.98, eps=1e-8, weight_decay=0.01)" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "6g_fEbV-72Hc" - }, - "source": [ - "Next, we will create the *training state* that includes the optimizer, the loss function, and is responsible for updating the model's parameters during training.\n", - "\n", - "Most JAX transformations (notably [jax.jit](https://jax.readthedocs.io/en/latest/jax-101/02-jitting.html)) require functions that are transformed to have no side effects. This is because any such side-effects will only be executed once when the Python version of the function is run during compilation (see [Stateful Computations in JAX](https://jax.readthedocs.io/en/latest/jax-101/07-state.html)). As a consequence, Flax models (which can be transformed by JAX transformations) are **immutable**, and the state of the model (i.e., its weight parameters) is stored *outside* of the model instance.\n", - "\n", - "Models are initialized and updated in a purely functional way: you pass the state to the model when calling it, and the model returns the new (possibly modified) state, leaving the model instance itself unchanged.\n", - "\n", - "Flax provides a convenience class [`flax.training.train_state.TrainState`](https://github.com/google/flax/blob/9da95cdd12591f42d2cd4c17089861bff7e43cc5/flax/training/train_state.py#L22), which stores things such as the model parameters, the loss function, the optimizer, and exposes an `apply_gradients` function to update the model's weight parameters.\n", - "\n", - "Alright, let's begin by defining our *training state* class. We create a `TrainState` class that stores the model's forward pass as the `apply_fn`, the `params`, and the AdamW optimizer." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "JHYfR67AoKRc" - }, - "source": [ - "state = train_state.TrainState.create(apply_fn=model.__call__, params=model.params, tx=adamw)" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "xiYCejDd81TX" - }, - "source": [ - "Next, let's implement a data loader for both training and evaluation.\n", - "The data loader can be defined as a [Python generator](https://wiki.python.org/moin/Generators) that returns a batch model input every time it is called.\n", - "\n", - "First, a random permutation of the whole dataset is defined. \n", - "Then, every time the training data collator is called the next batch of the randomized dataset is extracted, converted to a JAX array and sharded over all local TPU devices." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "Aos9GltTb3Ve" - }, - "source": [ - "def data_loader(rng, dataset, batch_size, shuffle=False):\n", - " steps_per_epoch = len(dataset) // batch_size\n", - "\n", - " if shuffle:\n", - " batch_idx = jax.random.permutation(rng, len(dataset))\n", - " else:\n", - " batch_idx = jnp.arange(len(dataset))\n", - "\n", - " batch_idx = batch_idx[: steps_per_epoch * batch_size] # Skip incomplete batch.\n", - " batch_idx = batch_idx.reshape((steps_per_epoch, batch_size))\n", - "\n", - " for idx in batch_idx:\n", - " batch = dataset[idx]\n", - " batch = {k: jnp.array(v) for k, v in batch.items()}\n", - "\n", - " batch = shard(batch)\n", - "\n", - " yield batch" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "L7uoTXDLUzb-" - }, - "source": [ - "At each training epoch, the dataset should be shuffled and superfluous samples that make the dataset not evenly divisible by the batch size are thrown away. Instead of passing the dataset, we prepare the indices of data samples to be used for both each training epoch. \n", - "The indices for the training dataset are additionally randomly shuffled before each epoch." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "MU6idLb29xYu" - }, - "source": [ - "During fine-tuning, we want to update the model parameters and evaluate the performance after each epoch. \n", - "\n", - "Let's write the functions `train_step` and `eval_step` accordingly. During training the weight parameters should be updated as follows:\n", - "\n", - "1. Define a loss function `loss_function` that first runs a forward pass of the model given data input. Remember that Flax models are immutable, and we explicitly pass it the state (in this case the model parameters and the RNG). `loss_function` returns a scalar loss (using the previously defined `state.loss_function`) between the model output and input targets.\n", - "2. Differentiate this loss function using [`jax.value_and_grad`](https://jax.readthedocs.io/en/latest/notebooks/autodiff_cookbook.html#evaluate-a-function-and-its-gradient-using-value-and-grad). This is a JAX transformation called [automatic differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation), which computes the gradient of `loss_function` given the input to the function (i.e., the parameters of the model), and returns the value and the gradient in a pair `(loss, gradients)`.\n", - "3. Compute the mean gradient over all devices using the collective operation [lax.pmean](https://jax.readthedocs.io/en/latest/_autosummary/jax.lax.pmean.html). As we will see below, each device runs `train_step` on a different batch of data, but by taking the mean here we ensure the model parameters are the same on all devices.\n", - "4. Use `state.apply_gradients`, which applies the gradients to the weights.\n", - "\n", - "Below, you can see how each of the described steps above is put into practice.\n", - "\n", - "Also note that the `labels` are shifted one to the left and the last token of the `logits` is cut. This way, the model learns to predict the **next** token as defined in causal language modeling." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "GjKzb0zJd-aH" - }, - "source": [ - "def train_step(state, batch, dropout_rng):\n", - " dropout_rng, new_dropout_rng = jax.random.split(dropout_rng)\n", - "\n", - " def loss_fn(params):\n", - " labels = batch.pop(\"labels\")\n", - " logits = state.apply_fn(**batch, params=params, dropout_rng=dropout_rng, train=True)[0]\n", - " \n", - " loss = optax.softmax_cross_entropy(logits[..., :-1, :], onehot(labels[..., 1:], logits.shape[-1])).mean()\n", - " return loss\n", - "\n", - " grad_fn = jax.value_and_grad(loss_fn)\n", - " loss, grad = grad_fn(state.params)\n", - " grad = jax.lax.pmean(grad, \"batch\")\n", - " new_state = state.apply_gradients(grads=grad)\n", - "\n", - " metrics = jax.lax.pmean(\n", - " {\"loss\": loss, \"learning_rate\": linear_decay_lr_schedule_fn(state.step)}, axis_name=\"batch\"\n", - " )\n", - "\n", - " return new_state, metrics, new_dropout_rng" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "nCPedI-B-FMQ" - }, - "source": [ - "Now, we want to do parallelized training over all TPU devices. To do so, we use [`jax.pmap`](https://jax.readthedocs.io/en/latest/jax.html?highlight=pmap#parallelization-pmap). This will compile the function once and run the same program on each device (it is an [SPMD program](https://en.wikipedia.org/wiki/SPMD)). When calling this pmapped function, all inputs (`\"state\"`, `\"batch\"`, `\"dropout_rng\"`) should be replicated for all devices, which means that the first axis of each argument is used to map over all TPU devices." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "w3k1Lqerpw5k" - }, - "source": [ - "parallel_train_step = jax.pmap(train_step, \"batch\")" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "0DWFAZM6A8uf" - }, - "source": [ - "Similarly, we can now define the evaluation step. Here, the function is much easier as we don't need to compute any gradients. To better monitor the performance improvement during training, the next token loss is computed and stored in a `metric` dictionary during evaluation." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "EGEv7dyfpW4p" - }, - "source": [ - "def eval_step(params, batch):\n", - " labels = batch.pop(\"labels\")\n", - "\n", - " logits = model(**batch, params=params, train=False)[0]\n", - "\n", - " loss = optax.softmax_cross_entropy(logits[..., :-1, :], onehot(labels[..., 1:], logits.shape[-1])).mean()\n", - "\n", - " # summarize metrics\n", - " metrics = {\"loss\": loss, \"perplexity\": jnp.exp(loss)}\n", - " metrics = jax.lax.pmean(metrics, axis_name=\"batch\")\n", - " return metrics" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "guaYWTvFA_66" - }, - "source": [ - "Similarly, we also apply `jax.pmap` to the evaluation step." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "0B8U2r2RpzjV" - }, - "source": [ - "parallel_eval_step = jax.pmap(eval_step, \"batch\")" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "DLaM60PCY8Ka" - }, - "source": [ - "Next, we replicate/copy the weight parameters on each device, so that we can pass them to our parallelized mapped functions." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "kncZTfALp3PG", - "outputId": "3ce8ee5a-7bda-4ba9-8774-c52a363a98f5" - }, - "source": [ - "state = flax.jax_utils.replicate(state)" - ], - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.7/dist-packages/jax/lib/xla_bridge.py:317: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.\n", - " \"jax.host_count has been renamed to jax.process_count. This alias \"\n", - "/usr/local/lib/python3.7/dist-packages/jax/lib/xla_bridge.py:304: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.\n", - " \"jax.host_id has been renamed to jax.process_index. This alias \"\n" - ], - "name": "stderr" - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "i2xg8oI-ZJ3P" - }, - "source": [ - "We can almost start training! In a final preparation step, we generate a seeded [**PRNGKey**](https://jax.readthedocs.io/en/latest/_autosummary/jax.random.PRNGKey.html#jax-random-prngkey) used as the random seed for dropout layers and dataset shuffling.\n", - "\n", - "Similar to how we had to copy/replicate the state on all 8 TPU devices, we also need to generate one `PRNGKey` per device, which is why we split the initial `rng` key into 8 random seeds. " - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "idu3E9ubqZH3" - }, - "source": [ - "rng = jax.random.PRNGKey(training_seed)\n", - "dropout_rngs = jax.random.split(rng, jax.local_device_count())" - ], - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "bKuMWHicbede" - }, - "source": [ - "Now, we are all set to finally start training! \n", - "Let's put all the pieces together and write the training loop. \n", - "\n", - "We start each epoch by generating a new random seed that will be used for dataset shuffling, the dropout layers and the input token masking. \n", - "\n", - "Next, we generate the training dataset indices.\n", - "In the first nested loop - the training loop - we shard the input batch on all 8 TPU devices, and run the training step. \n", - "\n", - "Analogs, in the second nested loop - the evaluation loop - the evaluation batches are sharded and the evaluation step is run.\n", - "\n", - "**Note**: It might seem that the following cell \"hangs\" when executed for the first time. This is because JAX first traces & compiles the code, the very first time it is run. After the first training step, you should notice that execution is much faster." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 427, - "referenced_widgets": [ - "8c156c360afb4ddc962b87577e093cc4", - "9808f288735b4f5d9eea7377d4603ccc", - "68681edff0db43559012b6653571a289", - "125e8d57908f4679824c96d5247f7119", - "8134ef53fa8044c99b537386e88dbfc9", - "9b35ba10c85048ce8044b87ab9948611", - "e246db4008b14a23b55d2820be3b1ed8", - "b08557f6936e4af9a23f659649979c8d", - "dac789bcf81e4aac86717438ed6e8bbe", - "a1452333389441eab2c8016cd019d2f5", - "799b382dd6c74d528cf77813f82a7335", - "51d0dfb67bbd47ff88964bb446a91967", - "940ce122551e42509e8cec0659e47c77", - "a4234dc426d3459a93263c8b5fe5a34b", - "3549bf51a354408b9a39cb4532957617", - "7f05c774880548d2b73e69fb847c0a2e", - "71fa85dc266149e28685b1f5252185a5", - "9cb943192a154dbc8c7961f128449bbd", - "9de6ea64b67c443fbce1fd6aef4a7409", - "9658cc6977bb4fdf8a081aa99c300872", - "07b26680343f42fc805a3b52b398a5f2", - "97216855bb7f483882afae68bf78a51d", - "a1b9aa80b73748f18e3938df073b3d22", - "ce70b524ffab4052912d47b8adee6748", - "3c5df3c942ca4c768926b431064be351", - "f2e66aebdb4d4812a1f61c1ff990390a", - "269cc543a69e4ba3bae7f0bef67f1e25", - "8cb8398bb3c94284ac2ff42c7b3bc7c5", - "0b898451f79c4b6eb1678a8f366a5cbf", - "7a5426d6898e4ba2b5e3c48a65096232", - "2cf7d4c7dc6f413eb8cc9460e5de8d93", - "f7851bd9702949ee8e6278ad69c9d981", - "ab08bd64073d4d488f9c41e8595bac08", - "5e1805a3c7f24c3f9f15e86d15462d44", - "fc2e1a72620742499457ab2e31da12a9", - "b3b4d96626044375bad11ba48cc25ee3", - "c55e4e5f6d0a42b4ac25fb6d68a8e842", - "c7cca560cd4340bb986297655f513746", - "cd4892863d524637b8e567e2beed23bb", - "c85760cb6ccb4020be999357f2501bf9", - "7d0775fcf6a8497db20326dc94bd2739", - "5215d791d9144c089b6996336c3085cd", - "839daa7d5cdc4669942f1b5ef5bf4903", - "4bc03fce15a14355b3429c138ea1eadd", - "d7cddd2cafe84d17829096b31ee5e93e", - "b4ce8892d5784011a649eeca7b17fd0d", - "7a7b598979174b338f8fdfc90af61e47", - "cd946e84023445d59eea9893fba02f1a", - "4ec581224b3f47f2acdfaeb740c94526", - "81c25bb635db4bc9a62636bf3a3868eb", - "1462263d23c340e781d596a3f5414e03", - "c8bd20a456cc4783a2fbbdec9a2880ea", - "f0aa35a704a84967a3471e5f698bb04c", - "f2c541b4ecd04729a518414f8b878099", - "402795b6506a45c9831fb0ddb2ce0749", - "0ba00c3340414029938c9c947d4a74b6", - "d17f610741f142b782c74ac4e04e6481", - "84c8a7cbfdb04690941f8c32299744bb", - "f3f0b212907843f5be2131b7b60a955d", - "0a6a241c99bb49ad9baaef052b157964", - "b642c546813240a499a165279c743e7a", - "6acb415dc8b947bb85ab8f8959bc10e1", - "2c311c4506264995b7fbaba02333e747", - "2fde17ab63e5444799ec1dc5501d946e", - "6173f15a3e5a40819bd4f3833544fd78", - "da2f69c01bcd4139980a84ddca297817", - "b9470519dfff45228e54d0181337ee17", - "b5824ca248b74052bb1696b47369838e", - "11f8b1ddd5af4bb3babd492bbaf3fb7d", - "861e2a6f8e3e4a5cacfd0a746180cc90", - "181f0680e5bb4d23a69f9cc747207883", - "0e2531c122b44823a012ba844066e60c", - "a2324cab4e9745ddad69b35e77ed148e", - "42c8cb98eaae4cee86d49c675d3a3604", - "146ce87ac5b445e385285bdbbd70fba9", - "46340cbdaf9c47719e005b42b7236be2", - "cd26f91d46ef42609d265f0b709c3401", - "6f5f003a277244b2824e09b877fefa74", - "57a93be7bb3d4167a89f7d73d5915551", - "db96a8d67e564cd58358916ddfb86948", - "194f5ce951d54193b887f1cc91e3640b", - "d8770c573383466f81d7dfd01f048677", - "1fe6ad92e6834142844bb32e7a0d424d", - "9d905b40bf2746e79abad7257cf8654f", - "bfa957f8482847978542373db16cfa8a", - "1f836daeb9b7495a9ae00bc2c28fa212", - "0973226c646f4bb28f291ed3eb4e5cba", - "fd72a18adfb34bc2b2dab769067dcc1a", - "91fb9675236949e6bb8c99015061647c", - "75f988a47d774cd38fd85977fda7a96c", - "503ff750d98d4f2c8f7af32adc221685", - "8b3645c111b842f4b1d9ff54bb94f26e", - "dabd19b76eec4139a3028aa1776ac496", - "b6410389dafe43bd885446a6d2d255b0", - "23187b20efd44965966d6576c0d057aa", - "6f876daff243410bb26b8feb1cf981e0", - "2374ba361eaa41899b7eabe7da829259", - "522331eac9cb49e4998384f42eeba912", - "6eaf7a1f2e5e4aa590da55adf969ecc6", - "29b0ec79b8134168a5e3ae382088c147", - "96f9368df9dc453495badb09e767d091", - "02b1a546bc334efa93ba8dff5890dd0c", - "b7f8df98ab5d45f39b34ceb1018c01b5", - "c251cf1c33c741049cc9710e89f09bc0", - "7e067f02bdb74846a42909cd9ad1961b", - "68cf981867f7461c999c9e9f85af6499", - "b6eec0df5ca64d5dbf32cb5231127e3b", - "6efef5f4637e4efaa373cb9214d4e911", - "d539e4c93f4941e9af0b742df4c53ae0", - "dafd837bebd44c00ae68641cc0b31d3d", - "47808ac98d064711bccc77037e53dade", - "acfab7adb0ed440b87fa08f81086522d", - "6fa186bd796b4d16bfb67657178e4f65", - "07fe952dec2b4f2687bcddec8701a70e", - "6c9e7145c2d7439db6f63b29c505e63e", - "43861346cadc460992ab7e2c30aa556c", - "6baec27867e943a2bad6e70e48978ea5", - "d2e74c9d62444e98b52078d6a15dd068", - "fcf96c9bb04642089466de2421bd1af2", - "b9d1f631577f4a2a8eb49d3f8535ba25", - "e9c8d09e028741979a764789fd7f256e", - "2bd15c15e3b748cd9ee001bb86c04b6d", - "f5e27446f61747e1a65e477e2975dbcf", - "1e17af39c928463b800d22ad1fe533e5", - "98e148811880459ea7a7641ee3a301b6", - "6669a1844b394ea499dc92e126132bfc", - "987310405403439893813fb8c5da94a1", - "f48e444c929748b2a875c6eecc028bc4", - "1ed01b7ba2a64bebbbf0e641c8d1560e", - "bfadf2c9e99c48e4860c2a67b2955120", - "63f47b7d6bc849d1b869367a6ba8b40f", - "1bfe0a0ab5694e1d857ea981090d4f09", - "7f6628d5f61f424ba6373da2f6defe76", - "01d1c1e078e7403daabd1e7d8d5a4fda", - "3b802a49c0824b968d88350f82ff8422", - "5a6ea875aeff4688ba37481af5b5d9fe", - "6dfceed81b9b4f8198a8186dfdc72d42", - "44ee8edfb7c744358f0cb4567535ce07", - "5eb4c37fcf6a4b618a8f573b837a8c3a", - "5bc32858674f442fa7b49efcccfc3c15", - "3ebb7960558247ec84ef155042d2077a", - "52a25fde354e45dd9635097d22c16def", - "ce3117149c7e4ae6a93fbe959bd3f543", - "900aa99ccfeb4a38a0ad90f8ce656fe6", - "b414ff8df761462e8df8f51788f69508", - "e0c2ef0fd4d544b9b80b00689d21f503", - "52cae29e50974a42a3882f91b2a8f2ba", - "02b454a4d7ce44b0b819fc7085b1d357", - "6321a346b671411791979a1a343fb494", - "22941d4a640b4d8bba1b97f723da0ef0", - "b6f9da40231f4404bb0c90458c94c5c2", - "7db664fc575747af8e10ad6f2783a166", - "62fd4952400246189945f2de7f0192c6", - "f07d424e031d457a98a88b94b0c844b8", - "a9f8af0a5e594c29b553983b6a1c4c7d", - "5829f975dedd45d3b315ab29c1b12253", - "2eb6756f48074993825b5bfac47641e1", - "389fcde31cf24c438a9fcdb155060495", - "a5a86539a58f4fe98ba96f9cd3d46588", - "32c067d64cb7445eb0464f3a10071e7c", - "c014bf31dccb4448b561a702361ed9bc", - "ef823903b36348978833920bb7e03881", - "c6855514c5a649ebadc2db53c267e9e0", - "8602122a90db462cacef9373fca054fe", - "892e02ac20ef4963871d915085139f7d", - "173704b44e2c472499bc4242ef9e0863", - "2f647e8c1be2464fb9c69e8af5e5bf5a", - "5fc3a90707f746c884d924e385b0cc1c" - ] - }, - "id": "U946A-YZp-Pe", - "outputId": "1fe21ffd-2e39-4470-fb87-6b11ab1d4024" - }, - "source": [ - "for epoch in tqdm(range(1, num_epochs + 1), desc=f\"Epoch ...\", position=0, leave=True):\n", - " rng, input_rng = jax.random.split(rng)\n", - "\n", - " # -- Train --\n", - " train_loader = data_loader(input_rng, tokenized_datasets[\"train\"], total_batch_size, shuffle=True)\n", - " with tqdm(total=len(tokenized_datasets[\"train\"]) // total_batch_size, desc=\"Training...\", leave=False) as progress_bar_train:\n", - " for model_inputs in train_loader:\n", - " # Model forward\n", - " state, train_metric, dropout_rngs = parallel_train_step(state, model_inputs, dropout_rngs)\n", - "\n", - " progress_bar_train.update(1)\n", - "\n", - " progress_bar_train.write(\n", - " f\"Train... ({epoch}/{num_epochs} | Loss: {round(train_metric['loss'].mean(), 3)}, Learning Rate: {round(train_metric['learning_rate'].mean(), 6)})\"\n", - " )\n", - "\n", - " # -- Eval --\n", - " eval_loader = data_loader(input_rng, tokenized_datasets[\"validation\"], total_batch_size)\n", - " eval_metrics = []\n", - " \n", - " with tqdm(total=len(tokenized_datasets[\"validation\"]) // total_batch_size, desc=\"Evaluation...\", leave=False) as progress_bar_eval:\n", - " for model_inputs in eval_loader:\n", - " # Model forward\n", - " eval_metric = parallel_eval_step(state.params, model_inputs)\n", - " eval_metrics.append(eval_metric)\n", - "\n", - " progress_bar_eval.update(1)\n", - " \n", - " eval_metrics = get_metrics(eval_metrics)\n", - " eval_metrics = jax.tree_map(jnp.mean, eval_metrics)\n", - " progress_bar_eval.write(\n", - " f\"Eval... ({epoch}/{num_epochs} | Loss: {eval_metrics['loss']} | Perplexity: {eval_metrics['perplexity']})\"\n", - " )" - ], - "execution_count": null, - "outputs": [ - { - "output_type": "display_data", - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "8c156c360afb4ddc962b87577e093cc4", - "version_minor": 0, - "version_major": 2 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Epoch ...', max=10.0, style=ProgressStyle(description_wid…" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "display_data", - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "dac789bcf81e4aac86717438ed6e8bbe", - "version_minor": 0, - "version_major": 2 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Training...', max=137.0, style=ProgressStyle(description_…" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "\r\r\rTrain... (1/10 | Loss: 6.935000419616699, Learning Rate: 0.0002699999895412475)\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "71fa85dc266149e28685b1f5252185a5", - "version_minor": 0, - "version_major": 2 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Evaluation...', max=12.0, style=ProgressStyle(description…" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "\r\r\rEval... (1/10 | Loss: 7.108445644378662 | Perplexity: 1246.529052734375)\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "3c5df3c942ca4c768926b431064be351", - "version_minor": 0, - "version_major": 2 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Training...', max=137.0, style=ProgressStyle(description_…" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "\r\r\rTrain... (2/10 | Loss: 6.334000110626221, Learning Rate: 0.00023999999393709004)\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "ab08bd64073d4d488f9c41e8595bac08", - "version_minor": 0, - "version_major": 2 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Evaluation...', max=12.0, style=ProgressStyle(description…" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "\r\r\rEval... (2/10 | Loss: 6.567610740661621 | Perplexity: 738.8753662109375)\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "7d0775fcf6a8497db20326dc94bd2739", - "version_minor": 0, - "version_major": 2 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Training...', max=137.0, style=ProgressStyle(description_…" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "\r\r\rTrain... (3/10 | Loss: 5.798000335693359, Learning Rate: 0.0002099999983329326)\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "4ec581224b3f47f2acdfaeb740c94526", - "version_minor": 0, - "version_major": 2 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Evaluation...', max=12.0, style=ProgressStyle(description…" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "\r\r\rEval... (3/10 | Loss: 6.278167247772217 | Perplexity: 557.9488525390625)\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "d17f610741f142b782c74ac4e04e6481", - "version_minor": 0, - "version_major": 2 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Training...', max=137.0, style=ProgressStyle(description_…" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "\r\r\rTrain... (4/10 | Loss: 5.557000160217285, Learning Rate: 0.00018000000272877514)\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "6173f15a3e5a40819bd4f3833544fd78", - "version_minor": 0, - "version_major": 2 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Evaluation...', max=12.0, style=ProgressStyle(description…" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "\r\r\rEval... (4/10 | Loss: 6.062875270843506 | Perplexity: 451.3289794921875)\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "a2324cab4e9745ddad69b35e77ed148e", - "version_minor": 0, - "version_major": 2 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Training...', max=137.0, style=ProgressStyle(description_…" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "\r\r\rTrain... (5/10 | Loss: 5.543000221252441, Learning Rate: 0.00014999999257270247)\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "194f5ce951d54193b887f1cc91e3640b", - "version_minor": 0, - "version_major": 2 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Evaluation...', max=12.0, style=ProgressStyle(description…" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "\r\r\rEval... (5/10 | Loss: 5.920379161834717 | Perplexity: 392.97332763671875)\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "91fb9675236949e6bb8c99015061647c", - "version_minor": 0, - "version_major": 2 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Training...', max=137.0, style=ProgressStyle(description_…" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "\r\r\rTrain... (6/10 | Loss: 5.361000061035156, Learning Rate: 0.00011999999696854502)\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "2374ba361eaa41899b7eabe7da829259", - "version_minor": 0, - "version_major": 2 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Evaluation...', max=12.0, style=ProgressStyle(description…" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "\r\r\rEval... (6/10 | Loss: 5.821027755737305 | Perplexity: 356.4353942871094)\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "7e067f02bdb74846a42909cd9ad1961b", - "version_minor": 0, - "version_major": 2 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Training...', max=137.0, style=ProgressStyle(description_…" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "\r\r\rTrain... (7/10 | Loss: 5.207000255584717, Learning Rate: 9.000000136438757e-05)\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "6fa186bd796b4d16bfb67657178e4f65", - "version_minor": 0, - "version_major": 2 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Evaluation...', max=12.0, style=ProgressStyle(description…" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "\r\r\rEval... (7/10 | Loss: 5.748736381530762 | Perplexity: 332.1453857421875)\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "e9c8d09e028741979a764789fd7f256e", - "version_minor": 0, - "version_major": 2 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Training...', max=137.0, style=ProgressStyle(description_…" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "\r\r\rTrain... (8/10 | Loss: 5.124000072479248, Learning Rate: 5.999999848427251e-05)\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "1ed01b7ba2a64bebbbf0e641c8d1560e", - "version_minor": 0, - "version_major": 2 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Evaluation...', max=12.0, style=ProgressStyle(description…" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "\r\r\rEval... (8/10 | Loss: 5.703180313110352 | Perplexity: 317.5106201171875)\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "6dfceed81b9b4f8198a8186dfdc72d42", - "version_minor": 0, - "version_major": 2 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Training...', max=137.0, style=ProgressStyle(description_…" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "\r\r\rTrain... (9/10 | Loss: 5.220000267028809, Learning Rate: 2.9999999242136255e-05)\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "b414ff8df761462e8df8f51788f69508", - "version_minor": 0, - "version_major": 2 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Evaluation...', max=12.0, style=ProgressStyle(description…" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "\r\r\rEval... (9/10 | Loss: 5.674434185028076 | Perplexity: 308.7478942871094)\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "62fd4952400246189945f2de7f0192c6", - "version_minor": 0, - "version_major": 2 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Training...', max=137.0, style=ProgressStyle(description_…" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "\r\r\rTrain... (10/10 | Loss: 4.992000102996826, Learning Rate: 0.0)\n" - ], - "name": "stdout" - }, - { - "output_type": "display_data", - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c014bf31dccb4448b561a702361ed9bc", - "version_minor": 0, - "version_major": 2 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, description='Evaluation...', max=12.0, style=ProgressStyle(description…" - ] - }, - "metadata": { - "tags": [] - } - }, - { - "output_type": "stream", - "text": [ - "\r\r\rEval... (10/10 | Loss: 5.66389274597168 | Perplexity: 305.58953857421875)\n", - "\n" - ], - "name": "stdout" - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ZI4XIhY-7hyh" - }, - "source": [ - "It can be seen that in this colab training already reaches a speed of 2.42 training steps per second. Executing [**`run_clm_flax.py`**](https://github.com/huggingface/transformers/tree/master/examples/flax/language-modeling/run_clm_flax.py) on a TPUv3-8 VM should be as fast as 7 training steps per second.\n", - "\n", - "For a more in-detail comparison of runtimes please refer to [this](https://github.com/huggingface/transformers/tree/master/examples/flax/language-modeling#runtime-evaluation) table." - ] - } - ] -} \ No newline at end of file diff --git a/EDA.ipynb b/EDA.ipynb deleted file mode 100644 index 76baf66..0000000 --- a/EDA.ipynb +++ /dev/null @@ -1,208 +0,0 @@ -{ - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - }, - "orig_nbformat": 4, - "kernelspec": { - "name": "python3", - "display_name": "Python 3.8.10 64-bit ('.venv': venv)" - }, - "interpreter": { - "hash": "25d32b45eddbc1e23c07b06a2c9ff49f418e028170a37fc806346e2c2002bf83" - } - }, - "nbformat": 4, - "nbformat_minor": 2, - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "None of PyTorch, TensorFlow >= 2.0, or Flax have been found. Models won't be available and only tokenizers, configuration and file/data utilities can be used.\n" - ] - } - ], - "source": [ - "import pickle\n", - "from collections import Counter\n", - "from uuid import uuid4\n", - "\n", - "from tqdm import tqdm\n", - "\n", - "from datasets import load_dataset\n", - "from transformers import AutoTokenizer" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "Using the latest cached version of the module from /home/shpotes/.cache/huggingface/modules/datasets_modules/datasets/code_clippy/86b09b4a623c1c39753a8ad165e05757d9a97daf132ac71d3b6eb791e7da16dd (last modified on Fri Jul 9 22:06:59 2021) since it couldn't be found locally at $HOME/gpt-code-clippy/code_clippy.py/code_clippy.py or remotely (FileNotFoundError).\n", - "Using the latest cached version of the module from /home/shpotes/.cache/huggingface/modules/datasets_modules/datasets/code_clippy/86b09b4a623c1c39753a8ad165e05757d9a97daf132ac71d3b6eb791e7da16dd (last modified on Fri Jul 9 22:06:59 2021) since it couldn't be found locally at $HOME/gpt-code-clippy/code_clippy.py/code_clippy.py or remotely (FileNotFoundError).\n", - "Using custom data configuration default-668fb26707140662\n", - "Using the latest cached version of the module from /home/shpotes/.cache/huggingface/modules/datasets_modules/datasets/code_clippy/86b09b4a623c1c39753a8ad165e05757d9a97daf132ac71d3b6eb791e7da16dd (last modified on Fri Jul 9 22:06:59 2021) since it couldn't be found locally at $HOME/gpt-code-clippy/code_clippy.py/code_clippy.py or remotely (FileNotFoundError).\n", - "Using the latest cached version of the module from /home/shpotes/.cache/huggingface/modules/datasets_modules/datasets/code_clippy/86b09b4a623c1c39753a8ad165e05757d9a97daf132ac71d3b6eb791e7da16dd (last modified on Fri Jul 9 22:06:59 2021) since it couldn't be found locally at $HOME/gpt-code-clippy/code_clippy.py/code_clippy.py or remotely (FileNotFoundError).\n", - "Using custom data configuration default-668fb26707140662\n" - ] - } - ], - "source": [ - "train_dataset = load_dataset(\n", - " \"$HOME/gpt-code-clippy/code_clippy.py\",\n", - " data_dir=\"/home/shared/code-clippy-dataset/merged-data\",\n", - " streaming=True,\n", - " split=\"train\"\n", - ")\n", - "\n", - "eval_dataset = load_dataset(\n", - " \"$HOME/gpt-code-clippy/code_clippy.py\",\n", - " data_dir=\"/home/shared/code-clippy-dataset/merged-data\",\n", - " streaming=True,\n", - " split=\"validation\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "tokenizer = AutoTokenizer.from_pretrained(\"EleutherAI/gpt-neo-125M\")" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "def _get_stats(example):\n", - " num_of_tokens = map(len, tokenizer(example[\"text\"])['input_ids'])\n", - " num_of_lines = map(lambda x: x.count(\"\\n\"), example[\"text\"])\n", - " file_name = map(lambda x: \".\".join(x.split(\".\")[:-1]), example[\"file_name\"])\n", - " langs = list(map(lambda x: x.split(\".\")[-1], example[\"file_name\"]))\n", - "\n", - " lang_map = {}\n", - " acc_tok = []\n", - " acc_lines = []\n", - " acc_fnames = []\n", - "\n", - " for tok, lines, fname, lang in zip(num_of_tokens, num_of_lines, file_name, langs):\n", - " if not lang in lang_map:\n", - " lang_idx = len(acc_tok)\n", - " lang_map[lang] = lang_idx\n", - "\n", - " acc_tok.append([tok])\n", - " acc_lines.append([lines])\n", - " acc_fnames.append([Counter({fname: 1})])\n", - " else:\n", - " lang_idx = lang_map[lang]\n", - "\n", - " acc_tok[lang_idx][0] += tok\n", - " acc_lines[lang_idx][0] += lines\n", - " acc_fnames[lang_idx][0].update({fname: 1})\n", - " \n", - " lang = [[k] for k, v in sorted(lang_map.items(), key=lambda item: item[1])]\n", - " _id = [str(uuid4())] * len(lang)\n", - "\n", - " return {\n", - " \"ext\": lang,\n", - " \"id\": _id,\n", - " \"acc_num_of_tokens\": acc_tok,\n", - " \"acc_num_of_lines\": acc_lines,\n", - " \"acc_file_names\": acc_fnames,\n", - " }" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - " def collapse_metrics(per_language_dataset, target_file_name):\n", - " num_of_tokens = {}\n", - " num_of_lines = {}\n", - " file_names = {}\n", - "\n", - " observed_lang = set()\n", - "\n", - " for row in tqdm(per_language_dataset):\n", - " lang = row['ext'][0]\n", - " if lang in observed_lang:\n", - " num_of_tokens[lang] += row['acc_num_of_tokens'][0]\n", - " num_of_lines[lang] += row['acc_num_of_lines'][0]\n", - " file_names[lang].update(row['acc_file_names'][0])\n", - " else:\n", - " num_of_tokens[lang] = row['acc_num_of_tokens'][0]\n", - " num_of_lines[lang] = row['acc_num_of_lines'][0]\n", - " file_names[lang] = row['acc_file_names'][0]\n", - "\n", - " with open(target_file_name, 'wb') as buf:\n", - " pickle.dump({\n", - " 'num_of_tokens': num_of_tokens,\n", - " 'num_of_lines': num_of_lines,\n", - " 'file_names': file_names,\n", - " }, buf)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "train_dataset = train_dataset.map(_get_stats, batched=True, batch_size=50_000)\n", - "eval_dataset = eval_dataset.map(_get_stats, batched=True, batch_size=50_000)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "0it [00:00, ?it/s]Token indices sequence length is longer than the specified maximum sequence length for this model (2708 > 2048). Running this sequence through the model will result in indexing errors\n", - "61it [00:55, 1.36it/s]" - ] - } - ], - "source": [ - "collapse_metrics(train_dataset, 'train_metrics.pkl')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "collapse_metrics(eval_dataset, 'eval_metrics.pkl')" - ] - } - ] -} \ No newline at end of file diff --git a/LICENSE b/LICENSE index 261eeb9..89ab696 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2021 Nathan Cooper and the amazing contributors of this project Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/scripts/check_repo_vulnerabilities.py b/data_processing/check_repo_vulnerabilities.py similarity index 100% rename from scripts/check_repo_vulnerabilities.py rename to data_processing/check_repo_vulnerabilities.py diff --git a/scripts/convert_to_gh_downloader_format.py b/data_processing/convert_to_gh_downloader_format.py similarity index 100% rename from scripts/convert_to_gh_downloader_format.py rename to data_processing/convert_to_gh_downloader_format.py diff --git a/deduplication/README.md b/data_processing/deduplication/README.md similarity index 100% rename from deduplication/README.md rename to data_processing/deduplication/README.md diff --git a/deduplication/deduplication.py b/data_processing/deduplication/deduplication.py similarity index 100% rename from deduplication/deduplication.py rename to data_processing/deduplication/deduplication.py diff --git a/deduplication/deduplication_parallel.py b/data_processing/deduplication/deduplication_parallel.py similarity index 100% rename from deduplication/deduplication_parallel.py rename to data_processing/deduplication/deduplication_parallel.py diff --git a/deduplication/deduplication_streaming.py b/data_processing/deduplication/deduplication_streaming.py similarity index 100% rename from deduplication/deduplication_streaming.py rename to data_processing/deduplication/deduplication_streaming.py diff --git a/deduplication/script.py b/data_processing/deduplication/script.py similarity index 100% rename from deduplication/script.py rename to data_processing/deduplication/script.py diff --git a/scripts/download_data.sh b/data_processing/download_data.sh similarity index 100% rename from scripts/download_data.sh rename to data_processing/download_data.sh diff --git a/scripts/scorecard b/data_processing/scorecard similarity index 100% rename from scripts/scorecard rename to data_processing/scorecard diff --git a/nbs/data_processing.ipynb b/evaluation/data_processing.ipynb similarity index 100% rename from nbs/data_processing.ipynb rename to evaluation/data_processing.ipynb diff --git a/nbs/gh-data-exploration.ipynb b/evaluation/gh-data-exploration.ipynb similarity index 100% rename from nbs/gh-data-exploration.ipynb rename to evaluation/gh-data-exploration.ipynb diff --git a/metrics/bleu.py b/evaluation/metrics/bleu.py similarity index 100% rename from metrics/bleu.py rename to evaluation/metrics/bleu.py diff --git a/metrics/extrinsic_eval.py b/evaluation/metrics/extrinsic_eval.py similarity index 100% rename from metrics/extrinsic_eval.py rename to evaluation/metrics/extrinsic_eval.py diff --git a/flax-gpt-neo-clm-v2.ipynb b/flax-gpt-neo-clm-v2.ipynb deleted file mode 100644 index 42ceaf9..0000000 --- a/flax-gpt-neo-clm-v2.ipynb +++ /dev/null @@ -1,6554 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "uGYl4nCPKyZi" - }, - "source": [ - "# Fine-tuning GPTNeo with 🤗 Transformers and **Flax/JAX** on TPU\n", - "\n", - "In this notebook, we will see how to pretrain one of the [🤗 Transformers](https://github.com/huggingface/transformers) models on TPU using [**Flax**](https://flax.readthedocs.io/en/latest/index.html). \n", - "\n", - "GPTNeo causal language modeling objective will be used for pre-training here.\n", - "\n", - "As can be seen on [this benchmark](https://github.com/huggingface/transformers/tree/master/examples/flax/language-modeling#runtime-evaluation) using Flax/JAX on GPU/TPU is often much faster and can also be considerably cheaper than using PyTorch on GPU/TPU.\n", - "\n", - "[**Flax**](https://flax.readthedocs.io/en/latest/index.html) is a high-performance neural network library designed for flexibility built on top of JAX (see below). It aims to provide users with full control of their training code and is carefully designed to work well with JAX transformations such as `grad` and `pmap` (see the [Flax philosophy](https://flax.readthedocs.io/en/latest/philosophy.html)). For an introduction to Flax see the [Flax Basic Colab](https://flax.readthedocs.io/en/latest/notebooks/flax_basics.html) or the list of curated [Flax examples](https://flax.readthedocs.io/en/latest/examples.html).\n", - "\n", - "[**JAX**](https://jax.readthedocs.io/en/latest/index.html) is Autograd and XLA, brought together for high-performance numerical computing and machine learning research. It provides composable transformations of Python+NumPy programs: differentiate, vectorize, parallelize, Just-In-Time compile to GPU/TPU, and more. A great place for getting started with JAX is the [JAX 101 Tutorial](https://jax.readthedocs.io/en/latest/jax-101/index.html)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "PwDAzFXQMd46" - }, - "source": [ - "If you're opening this Notebook on colab, you will probably need to install 🤗 Transformers, 🤗 Datasets, 🤗 Tokenizers as well as [Flax](https://github.com/google/flax.git) and [Optax](https://github.com/deepmind/optax). Optax is a gradient processing and optimization library for JAX, and is the optimizer library\n", - "recommended by Flax." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "If_SYBvU5V6u" - }, - "source": [ - "If everything is set up correctly, the following command should return a list of 8 TPU devices." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "!export HF_HOME=\"/home/shared/.cache/hf/\"" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "3R5MP7PAbV7V", - "outputId": "9cf6b9a4-7b9c-4029-d938-dcf9a3ebb4c4" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "[TpuDevice(id=0, process_index=0, coords=(0,0,0), core_on_chip=0),\n", - " TpuDevice(id=1, process_index=0, coords=(0,0,0), core_on_chip=1),\n", - " TpuDevice(id=2, process_index=0, coords=(1,0,0), core_on_chip=0),\n", - " TpuDevice(id=3, process_index=0, coords=(1,0,0), core_on_chip=1),\n", - " TpuDevice(id=4, process_index=0, coords=(0,1,0), core_on_chip=0),\n", - " TpuDevice(id=5, process_index=0, coords=(0,1,0), core_on_chip=1),\n", - " TpuDevice(id=6, process_index=0, coords=(1,1,0), core_on_chip=0),\n", - " TpuDevice(id=7, process_index=0, coords=(1,1,0), core_on_chip=1)]" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import jax\n", - "jax.local_devices()" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "id": "ii9XwLsmiY-E" - }, - "outputs": [], - "source": [ - "ds_name = \"code_search_net\"\n", - "language = \"python\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jVtv6T0oSjNq" - }, - "source": [ - "Next, we select the model architecture to be trained from scratch.\n", - "Here we choose [**`distilgpt2`**](https://huggingface.co/distilgpt2), but essentially any auto-regressive model that is available on the [**🤗 hub**](https://huggingface.co/models?filter=masked-lm,jax) in JAX/Flax can be used. " - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "from ipywidgets import Dropdown" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c10c577b3949433ca2fd33373d2e211f", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Dropdown(options=('EleutherAI/gpt-neo-125M', 'EleutherAI/gpt-neo-1.3B', 'EleutherAI/gpt-neo-2.7B'), value='Ele…" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "ckpt_selector = Dropdown(options = [\"EleutherAI/gpt-neo-125M\", \"EleutherAI/gpt-neo-1.3B\", \"EleutherAI/gpt-neo-2.7B\"])\n", - "ckpt_selector" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "id": "Sj1mJNJa6PPS" - }, - "outputs": [], - "source": [ - "model_ckpt = ckpt_selector.value" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "id": "1dwuSvQxeM8-" - }, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "output_dir = Path.home()/f\"{model_ckpt.split('/')[1]}-code-clippy\"" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2021-07-05 16:10:01.811521: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n" - ] - } - ], - "source": [ - "import logging\n", - "import math\n", - "import os\n", - "import sys\n", - "import time\n", - "from dataclasses import dataclass, field\n", - "from pathlib import Path\n", - "from typing import Callable, Optional\n", - "\n", - "import datasets\n", - "from datasets import Dataset, load_dataset\n", - "from tqdm.auto import tqdm\n", - "\n", - "import jax\n", - "import jax.numpy as jnp\n", - "import optax\n", - "import transformers\n", - "from flax import jax_utils, traverse_util\n", - "from flax.jax_utils import unreplicate\n", - "from flax.training import train_state\n", - "from flax.training.common_utils import get_metrics, onehot, shard, shard_prng_key\n", - "from transformers import (\n", - " CONFIG_MAPPING,\n", - " FLAX_MODEL_FOR_CAUSAL_LM_MAPPING,\n", - " AutoConfig,\n", - " AutoTokenizer,\n", - " FlaxAutoModelForCausalLM,\n", - " HfArgumentParser,\n", - " TrainingArguments,\n", - " is_tensorboard_available,\n", - ")\n", - "from transformers.testing_utils import CaptureLogger" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "# import jax.profiler\n", - "# server = jax.profiler.start_server(9999)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "logger = logging.getLogger(__name__)\n", - "\n", - "# Cache the result\n", - "has_tensorboard = is_tensorboard_available()\n", - "if has_tensorboard:\n", - " try:\n", - " from flax.metrics.tensorboard import SummaryWriter\n", - " except ImportError as ie:\n", - " has_tensorboard = False\n", - " print(f\"Unable to display metrics through TensorBoard because some package are not installed: {ie}\")\n", - "\n", - "else:\n", - " print(\n", - " \"Unable to display metrics through TensorBoard because the package is not installed: \"\n", - " \"Please run pip install tensorboard to enable.\"\n", - " )\n", - "\n", - "\n", - "MODEL_CONFIG_CLASSES = list(FLAX_MODEL_FOR_CAUSAL_LM_MAPPING.keys())\n", - "MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "@dataclass\n", - "class ModelArguments:\n", - " \"\"\"\n", - " Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch.\n", - " \"\"\"\n", - "\n", - " model_name_or_path: Optional[str] = field(\n", - " default=None,\n", - " metadata={\n", - " \"help\": \"The model checkpoint for weights initialization.\"\n", - " \"Don't set if you want to train a model from scratch.\"\n", - " },\n", - " )\n", - " model_type: Optional[str] = field(\n", - " default=None,\n", - " metadata={\"help\": \"If training from scratch, pass a model type from the list: \" + \", \".join(MODEL_TYPES)},\n", - " )\n", - " config_name: Optional[str] = field(\n", - " default=None, metadata={\"help\": \"Pretrained config name or path if not the same as model_name\"}\n", - " )\n", - " tokenizer_name: Optional[str] = field(\n", - " default=None, metadata={\"help\": \"Pretrained tokenizer name or path if not the same as model_name\"}\n", - " )\n", - " cache_dir: Optional[str] = field(\n", - " default=None, metadata={\"help\": \"Where do you want to store the pretrained models downloaded from s3\"}\n", - " )\n", - " use_fast_tokenizer: bool = field(\n", - " default=True,\n", - " metadata={\"help\": \"Whether to use one of the fast tokenizer (backed by the tokenizers library) or not.\"},\n", - " )\n", - " dtype: Optional[str] = field(\n", - " default=\"float32\",\n", - " metadata={\n", - " \"help\": \"Floating-point format in which the model weights should be initialized and trained. Choose one of `[float32, float16, bfloat16]`.\"\n", - " },\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "@dataclass\n", - "class DataTrainingArguments:\n", - " \"\"\"\n", - " Arguments pertaining to what data we are going to input our model for training and eval.\n", - " \"\"\"\n", - "\n", - " dataset_name: Optional[str] = field(\n", - " default=None, metadata={\"help\": \"The name of the dataset to use (via the datasets library).\"}\n", - " )\n", - " dataset_config_name: Optional[str] = field(\n", - " default=None, metadata={\"help\": \"The configuration name of the dataset to use (via the datasets library).\"}\n", - " )\n", - " train_file: Optional[str] = field(default=None, metadata={\"help\": \"The input training data file (a text file).\"})\n", - " validation_file: Optional[str] = field(\n", - " default=None,\n", - " metadata={\"help\": \"An optional input evaluation data file to evaluate the perplexity on (a text file).\"},\n", - " )\n", - " max_train_samples: Optional[int] = field(\n", - " default=None,\n", - " metadata={\n", - " \"help\": \"For debugging purposes or quicker training, truncate the number of training examples to this \"\n", - " \"value if set.\"\n", - " },\n", - " )\n", - " max_eval_samples: Optional[int] = field(\n", - " default=None,\n", - " metadata={\n", - " \"help\": \"For debugging purposes or quicker training, truncate the number of evaluation examples to this \"\n", - " \"value if set.\"\n", - " },\n", - " )\n", - " overwrite_cache: bool = field(\n", - " default=False, metadata={\"help\": \"Overwrite the cached training and evaluation sets\"}\n", - " )\n", - " validation_split_percentage: Optional[int] = field(\n", - " default=5,\n", - " metadata={\n", - " \"help\": \"The percentage of the train set used as validation set in case there's no validation split\"\n", - " },\n", - " )\n", - " block_size: Optional[int] = field(\n", - " default=None,\n", - " metadata={\n", - " \"help\": \"Optional input sequence length after tokenization. \"\n", - " \"The training dataset will be truncated in block of this size for training. \"\n", - " \"Default to the model max input length for single sentence inputs (take into account special tokens).\"\n", - " },\n", - " )\n", - " overwrite_cache: bool = field(\n", - " default=False, metadata={\"help\": \"Overwrite the cached training and evaluation sets\"}\n", - " )\n", - " preprocessing_num_workers: Optional[int] = field(\n", - " default=None,\n", - " metadata={\"help\": \"The number of processes to use for the preprocessing.\"},\n", - " )\n", - "\n", - " def __post_init__(self):\n", - " if self.dataset_name is None and self.train_file is None and self.validation_file is None:\n", - " raise ValueError(\"Need either a dataset name or a training/validation file.\")\n", - " else:\n", - " if self.train_file is not None:\n", - " extension = self.train_file.split(\".\")[-1]\n", - " assert extension in [\"csv\", \"json\", \"txt\"], \"`train_file` should be a csv, a json or a txt file.\"\n", - " if self.validation_file is not None:\n", - " extension = self.validation_file.split(\".\")[-1]\n", - " assert extension in [\"csv\", \"json\", \"txt\"], \"`validation_file` should be a csv, a json or a txt file.\"" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "class TrainState(train_state.TrainState):\n", - " dropout_rng: jnp.ndarray\n", - "\n", - " def replicate(self):\n", - " return jax_utils.replicate(self).replace(dropout_rng=shard_prng_key(self.dropout_rng))" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "def data_loader(rng: jax.random.PRNGKey, dataset: Dataset, batch_size: int, shuffle: bool = False):\n", - " \"\"\"\n", - " Returns batches of size `batch_size` from truncated `dataset`, sharded over all local devices.\n", - " Shuffle batches if `shuffle` is `True`.\n", - " \"\"\"\n", - " steps_per_epoch = len(dataset) // batch_size\n", - "\n", - " if shuffle:\n", - " batch_idx = jax.random.permutation(rng, len(dataset))\n", - " else:\n", - " batch_idx = jnp.arange(len(dataset))\n", - "\n", - " batch_idx = batch_idx[: steps_per_epoch * batch_size] # Skip incomplete batch.\n", - " batch_idx = batch_idx.reshape((steps_per_epoch, batch_size))\n", - "\n", - " for idx in batch_idx:\n", - " batch = dataset[idx]\n", - " batch = {k: jnp.array(v) for k, v in batch.items()}\n", - "\n", - " batch = shard(batch)\n", - "\n", - " yield batch" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "def write_metric(summary_writer, train_metrics, eval_metrics, train_time, step):\n", - " summary_writer.scalar(\"train_time\", train_time, step)\n", - "\n", - " train_metrics = get_metrics(train_metrics)\n", - " for key, vals in train_metrics.items():\n", - " tag = f\"train_{key}\"\n", - " for i, val in enumerate(vals):\n", - " summary_writer.scalar(tag, val, step - len(vals) + i + 1)\n", - "\n", - " for metric_name, value in eval_metrics.items():\n", - " summary_writer.scalar(f\"eval_{metric_name}\", value, step)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "def create_learning_rate_fn(\n", - " train_ds_size: int, train_batch_size: int, num_train_epochs: int, warmup_ratio: float, learning_rate: float\n", - ") -> Callable[[int], jnp.array]:\n", - " \"\"\"Returns a linear warmup, linear_decay learning rate function.\"\"\"\n", - " steps_per_epoch = train_ds_size // train_batch_size\n", - " num_train_steps = steps_per_epoch * num_train_epochs\n", - " num_warmup_steps = int(num_train_steps * warmup_ratio)\n", - " warmup_fn = optax.linear_schedule(init_value=0.0, end_value=learning_rate, transition_steps=num_warmup_steps)\n", - " decay_fn = optax.linear_schedule(\n", - " init_value=learning_rate, end_value=0, transition_steps=num_train_steps - num_warmup_steps\n", - " )\n", - " schedule_fn = optax.join_schedules(schedules=[warmup_fn, decay_fn], boundaries=[num_warmup_steps])\n", - " return schedule_fn" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Set up training arguments" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "model_args = ModelArguments(\n", - " model_name_or_path=model_ckpt,\n", - " dtype=\"bfloat16\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "data_args = DataTrainingArguments(\n", - " dataset_name=ds_name, \n", - " dataset_config_name=language, \n", - " block_size=512,\n", - "# max_train_samples=20000, \n", - "# max_eval_samples=2000, \n", - " preprocessing_num_workers=8\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [], - "source": [ - "bs = 32\n", - "training_args = TrainingArguments(\n", - " num_train_epochs=1,\n", - " output_dir=output_dir, \n", - " per_device_train_batch_size=bs, \n", - " per_device_eval_batch_size=bs, \n", - " learning_rate=3e-4,\n", - " weight_decay=0.1,\n", - " do_train=True,\n", - " do_eval=True,\n", - " warmup_ratio=0.1,\n", - " push_to_hub=False,\n", - " overwrite_output_dir=True\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [], - "source": [ - "if (\n", - " os.path.exists(training_args.output_dir)\n", - " and os.listdir(training_args.output_dir)\n", - " and training_args.do_train\n", - " and not training_args.overwrite_output_dir\n", - "):\n", - " raise ValueError(\n", - " f\"Output directory ({training_args.output_dir}) already exists and is not empty.\"\n", - " \"Use --overwrite_output_dir to overcome.\"\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:__main__:Training/evaluation parameters TrainingArguments(\n", - "_n_gpu=0,\n", - "adafactor=False,\n", - "adam_beta1=0.9,\n", - "adam_beta2=0.999,\n", - "adam_epsilon=1e-08,\n", - "dataloader_drop_last=False,\n", - "dataloader_num_workers=0,\n", - "dataloader_pin_memory=True,\n", - "ddp_find_unused_parameters=None,\n", - "debug=[],\n", - "deepspeed=None,\n", - "disable_tqdm=False,\n", - "do_eval=True,\n", - "do_predict=False,\n", - "do_train=True,\n", - "eval_accumulation_steps=None,\n", - "eval_steps=500,\n", - "evaluation_strategy=IntervalStrategy.NO,\n", - "fp16=False,\n", - "fp16_backend=auto,\n", - "fp16_full_eval=False,\n", - "fp16_opt_level=O1,\n", - "gradient_accumulation_steps=1,\n", - "greater_is_better=None,\n", - "group_by_length=False,\n", - "ignore_data_skip=False,\n", - "label_names=None,\n", - "label_smoothing_factor=0.0,\n", - "learning_rate=0.0003,\n", - "length_column_name=length,\n", - "load_best_model_at_end=False,\n", - "local_rank=-1,\n", - "log_level=-1,\n", - "log_level_replica=-1,\n", - "log_on_each_node=True,\n", - "logging_dir=/home/arto/gpt-neo-125M-code-clippy/runs/Jul05_16-10-02_t1v-n-416475e4-w-0,\n", - "logging_first_step=False,\n", - "logging_steps=500,\n", - "logging_strategy=IntervalStrategy.STEPS,\n", - "lr_scheduler_type=SchedulerType.LINEAR,\n", - "max_grad_norm=1.0,\n", - "max_steps=-1,\n", - "metric_for_best_model=None,\n", - "mp_parameters=,\n", - "no_cuda=False,\n", - "num_train_epochs=1,\n", - "output_dir=/home/arto/gpt-neo-125M-code-clippy,\n", - "overwrite_output_dir=True,\n", - "past_index=-1,\n", - "per_device_eval_batch_size=32,\n", - "per_device_train_batch_size=32,\n", - "prediction_loss_only=False,\n", - "push_to_hub=False,\n", - "push_to_hub_model_id=gpt-neo-125M-code-clippy,\n", - "push_to_hub_organization=None,\n", - "push_to_hub_token=None,\n", - "remove_unused_columns=True,\n", - "report_to=['tensorboard'],\n", - "resume_from_checkpoint=None,\n", - "run_name=/home/arto/gpt-neo-125M-code-clippy,\n", - "save_on_each_node=False,\n", - "save_steps=500,\n", - "save_strategy=IntervalStrategy.STEPS,\n", - "save_total_limit=None,\n", - "seed=42,\n", - "sharded_ddp=[],\n", - "skip_memory_metrics=True,\n", - "tpu_metrics_debug=False,\n", - "tpu_num_cores=None,\n", - "use_legacy_prediction_loop=False,\n", - "warmup_ratio=0.1,\n", - "warmup_steps=0,\n", - "weight_decay=0.1,\n", - ")\n" - ] - } - ], - "source": [ - "# Make one log on every process with the configuration for debugging.\n", - "logging.basicConfig(\n", - " format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n", - " datefmt=\"%m/%d/%Y %H:%M:%S\",\n", - " level=logging.INFO,\n", - ")\n", - "# Setup logging, we only want one process per machine to log things on the screen.\n", - "logger.setLevel(logging.INFO if jax.process_index() == 0 else logging.ERROR)\n", - "if jax.process_index() == 0:\n", - " datasets.utils.logging.set_verbosity_warning()\n", - " transformers.utils.logging.set_verbosity_info()\n", - "else:\n", - " datasets.utils.logging.set_verbosity_error()\n", - " transformers.utils.logging.set_verbosity_error()\n", - "\n", - "# Set the verbosity to info of the Transformers logger (on main process only):\n", - "logger.info(f\"Training/evaluation parameters {training_args}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:datasets.builder:Reusing dataset code_search_net (/home/arto/.cache/huggingface/datasets/code_search_net/python/1.0.0/80a244ab541c6b2125350b764dc5c2b715f65f00de7a56107a28915fac173a27)\n" - ] - } - ], - "source": [ - "# Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)\n", - "# or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/\n", - "# (the dataset will be downloaded automatically from the datasets Hub).\n", - "#\n", - "# For CSV/JSON files, this script will use the column called 'text' or the first column if no column called\n", - "# 'text' is found. You can easily tweak this behavior (see below).\n", - "#\n", - "# In distributed training, the load_dataset function guarantees that only one local process can concurrently\n", - "# download the dataset.\n", - "if data_args.dataset_name is not None:\n", - " # Downloading and loading a dataset from the hub.\n", - " dataset = load_dataset(\n", - " data_args.dataset_name, data_args.dataset_config_name, cache_dir=model_args.cache_dir, keep_in_memory=False\n", - " )\n", - "\n", - " if \"validation\" not in dataset.keys():\n", - " dataset[\"validation\"] = load_dataset(\n", - " data_args.dataset_name,\n", - " data_args.dataset_config_name,\n", - " split=f\"train[:{data_args.validation_split_percentage}%]\",\n", - " cache_dir=model_args.cache_dir,\n", - " )\n", - " dataset[\"train\"] = load_dataset(\n", - " data_args.dataset_name,\n", - " data_args.dataset_config_name,\n", - " split=f\"train[{data_args.validation_split_percentage}%:]\",\n", - " cache_dir=model_args.cache_dir,\n", - " )\n", - "else:\n", - " data_files = {}\n", - " if data_args.train_file is not None:\n", - " data_files[\"train\"] = data_args.train_file\n", - " if data_args.validation_file is not None:\n", - " data_files[\"validation\"] = data_args.validation_file\n", - " extension = data_args.train_file.split(\".\")[-1]\n", - " if extension == \"txt\":\n", - " extension = \"text\"\n", - " dataset = load_dataset(extension, data_files=data_files, cache_dir=model_args.cache_dir)\n", - "# See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at\n", - "# https://huggingface.co/docs/datasets/loading_datasets.html." - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "loading configuration file https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/config.json from cache at /home/arto/.cache/huggingface/transformers/29380fef22a43cbfb3d3a6c8e2f4fd951459584d87c34e4621b30580a54aca84.f0f7ebddfc6e15a23ac33e7fa95cd8cca05edf87cc74f9e3be7905f538a59762\n", - "Model config GPTNeoConfig {\n", - " \"activation_function\": \"gelu_new\",\n", - " \"architectures\": [\n", - " \"GPTNeoForCausalLM\"\n", - " ],\n", - " \"attention_dropout\": 0,\n", - " \"attention_layers\": [\n", - " \"global\",\n", - " \"local\",\n", - " \"global\",\n", - " \"local\",\n", - " \"global\",\n", - " \"local\",\n", - " \"global\",\n", - " \"local\",\n", - " \"global\",\n", - " \"local\",\n", - " \"global\",\n", - " \"local\"\n", - " ],\n", - " \"attention_types\": [\n", - " [\n", - " [\n", - " \"global\",\n", - " \"local\"\n", - " ],\n", - " 6\n", - " ]\n", - " ],\n", - " \"bos_token_id\": 50256,\n", - " \"embed_dropout\": 0,\n", - " \"eos_token_id\": 50256,\n", - " \"gradient_checkpointing\": false,\n", - " \"hidden_size\": 768,\n", - " \"initializer_range\": 0.02,\n", - " \"intermediate_size\": null,\n", - " \"layer_norm_epsilon\": 1e-05,\n", - " \"max_position_embeddings\": 2048,\n", - " \"model_type\": \"gpt_neo\",\n", - " \"num_heads\": 12,\n", - " \"num_layers\": 12,\n", - " \"resid_dropout\": 0,\n", - " \"summary_activation\": null,\n", - " \"summary_first_dropout\": 0.1,\n", - " \"summary_proj_to_labels\": true,\n", - " \"summary_type\": \"cls_index\",\n", - " \"summary_use_proj\": true,\n", - " \"transformers_version\": \"4.9.0.dev0\",\n", - " \"use_cache\": true,\n", - " \"vocab_size\": 50257,\n", - " \"window_size\": 256\n", - "}\n", - "\n", - "loading configuration file https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/config.json from cache at /home/arto/.cache/huggingface/transformers/29380fef22a43cbfb3d3a6c8e2f4fd951459584d87c34e4621b30580a54aca84.f0f7ebddfc6e15a23ac33e7fa95cd8cca05edf87cc74f9e3be7905f538a59762\n", - "Model config GPTNeoConfig {\n", - " \"activation_function\": \"gelu_new\",\n", - " \"architectures\": [\n", - " \"GPTNeoForCausalLM\"\n", - " ],\n", - " \"attention_dropout\": 0,\n", - " \"attention_layers\": [\n", - " \"global\",\n", - " \"local\",\n", - " \"global\",\n", - " \"local\",\n", - " \"global\",\n", - " \"local\",\n", - " \"global\",\n", - " \"local\",\n", - " \"global\",\n", - " \"local\",\n", - " \"global\",\n", - " \"local\"\n", - " ],\n", - " \"attention_types\": [\n", - " [\n", - " [\n", - " \"global\",\n", - " \"local\"\n", - " ],\n", - " 6\n", - " ]\n", - " ],\n", - " \"bos_token_id\": 50256,\n", - " \"embed_dropout\": 0,\n", - " \"eos_token_id\": 50256,\n", - " \"gradient_checkpointing\": false,\n", - " \"hidden_size\": 768,\n", - " \"initializer_range\": 0.02,\n", - " \"intermediate_size\": null,\n", - " \"layer_norm_epsilon\": 1e-05,\n", - " \"max_position_embeddings\": 2048,\n", - " \"model_type\": \"gpt_neo\",\n", - " \"num_heads\": 12,\n", - " \"num_layers\": 12,\n", - " \"resid_dropout\": 0,\n", - " \"summary_activation\": null,\n", - " \"summary_first_dropout\": 0.1,\n", - " \"summary_proj_to_labels\": true,\n", - " \"summary_type\": \"cls_index\",\n", - " \"summary_use_proj\": true,\n", - " \"transformers_version\": \"4.9.0.dev0\",\n", - " \"use_cache\": true,\n", - " \"vocab_size\": 50257,\n", - " \"window_size\": 256\n", - "}\n", - "\n", - "loading file https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/vocab.json from cache at /home/arto/.cache/huggingface/transformers/08c00c4159e921d4c941ac75732643373aba509d9b352a82bbbb043a94058d98.a552555fdda56a1c7c9a285bccfd44ac8e4b9e26c8c9b307831b3ea3ac782b45\n", - "loading file https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/merges.txt from cache at /home/arto/.cache/huggingface/transformers/12305762709d884a770efe7b0c68a7f4bc918da44e956058d43da0d12f7bea20.5d12962c5ee615a4c803841266e9c3be9a691a924f72d395d3a6c6c81157788b\n", - "loading file https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/tokenizer.json from cache at None\n", - "loading file https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/added_tokens.json from cache at None\n", - "loading file https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/special_tokens_map.json from cache at /home/arto/.cache/huggingface/transformers/6c3239a63aaf46ec7625b38abfe41fc2ce0b25f90800aefe6526256340d4ab6d.2b8bf81243d08385c806171bc7ced6d2a0dcc7f896ca637f4e777418f7f0cc3c\n", - "loading file https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/tokenizer_config.json from cache at /home/arto/.cache/huggingface/transformers/3cc88b3aa29bb2546db2dc21783292e2a086bb7158c7b5ceddeb24158a85c183.e74f7c3643ee79eb023ead36008be72fe726dada60fa3b2a0569925cfefa1e74\n", - "loading weights file https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/flax_model.msgpack from cache at /home/arto/.cache/huggingface/transformers/86b67f0b86eaa4932dabbbc6f42b33001baa8a069569d4d556d6f6754a43566d.77f87bf1f13f66c14b127d42fb39f9d4fff2e3b5679330a5a0efd6e91c91ce6c\n", - "All model checkpoint weights were used when initializing FlaxGPTNeoForCausalLM.\n", - "\n", - "All the weights of FlaxGPTNeoForCausalLM were initialized from the model checkpoint at EleutherAI/gpt-neo-125M.\n", - "If your task is similar to the task the model of the checkpoint was trained on, you can already use FlaxGPTNeoForCausalLM for predictions without further training.\n" - ] - } - ], - "source": [ - "# Load pretrained model and tokenizer\n", - "\n", - "# Distributed training:\n", - "# The .from_pretrained methods guarantee that only one local process can concurrently\n", - "# download model & vocab.\n", - "if model_args.config_name:\n", - " config = AutoConfig.from_pretrained(model_args.config_name, cache_dir=model_args.cache_dir)\n", - "elif model_args.model_name_or_path:\n", - " config = AutoConfig.from_pretrained(model_args.model_name_or_path, cache_dir=model_args.cache_dir)\n", - "else:\n", - " config = CONFIG_MAPPING[model_args.model_type]()\n", - " logger.warning(\"You are instantiating a new config instance from scratch.\")\n", - "\n", - "if model_args.tokenizer_name:\n", - " tokenizer = AutoTokenizer.from_pretrained(\n", - " model_args.tokenizer_name, cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer\n", - " )\n", - "elif model_args.model_name_or_path:\n", - " tokenizer = AutoTokenizer.from_pretrained(\n", - " model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer\n", - " )\n", - "else:\n", - " raise ValueError(\n", - " \"You are instantiating a new tokenizer from scratch. This is not supported by this script.\"\n", - " \"You can do it from another script, save it, and load it from here, using --tokenizer_name.\"\n", - " )\n", - "\n", - "if model_args.model_name_or_path:\n", - " model = FlaxAutoModelForCausalLM.from_pretrained(\n", - " model_args.model_name_or_path, config=config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype)\n", - " )\n", - "else:\n", - " model = FlaxAutoModelForCausalLM.from_config(\n", - " config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype)\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Token indices sequence length is longer than the specified maximum sequence length for this model (2397 > 2048). Running this sequence through the model will result in indexing errors\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (3042 > 2048). Running this sequence through the model will result in indexing errors\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (2571 > 2048). Running this sequence through the model will result in indexing errors\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (2090 > 2048). Running this sequence through the model will result in indexing errors\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (4783 > 2048). Running this sequence through the model will result in indexing errors\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (3104 > 2048). Running this sequence through the model will result in indexing errors\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (3366 > 2048). Running this sequence through the model will result in indexing errors\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (3897 > 2048). Running this sequence through the model will result in indexing errors\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (2272 > 2048). Running this sequence through the model will result in indexing errors\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (2321 > 2048). Running this sequence through the model will result in indexing errors\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (7101 > 2048). Running this sequence through the model will result in indexing errors\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (6130 > 2048). Running this sequence through the model will result in indexing errors\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (3291 > 2048). Running this sequence through the model will result in indexing errors\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (2492 > 2048). Running this sequence through the model will result in indexing errors\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (2694 > 2048). Running this sequence through the model will result in indexing errors\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (2402 > 2048). Running this sequence through the model will result in indexing errors\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (2559 > 2048). Running this sequence through the model will result in indexing errors\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (3021 > 2048). Running this sequence through the model will result in indexing errors\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (3446 > 2048). Running this sequence through the model will result in indexing errors\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (3679 > 2048). Running this sequence through the model will result in indexing errors\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (3164 > 2048). Running this sequence through the model will result in indexing errors\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (2129 > 2048). Running this sequence through the model will result in indexing errors\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (4533 > 2048). Running this sequence through the model will result in indexing errors\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (3615 > 2048). Running this sequence through the model will result in indexing errors\n", - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n" - ] - } - ], - "source": [ - "# Preprocessing the datasets.\n", - "# First we tokenize all the texts.\n", - "if training_args.do_train:\n", - " column_names = dataset[\"train\"].column_names\n", - "else:\n", - " column_names = dataset[\"validation\"].column_names\n", - "text_column_name = \"func_code_string\"#\"text\" if \"text\" in column_names else column_names[0]\n", - "\n", - "# since this will be pickled to avoid _LazyModule error in Hasher force logger loading before tokenize_function\n", - "tok_logger = transformers.utils.logging.get_logger(\"transformers.tokenization_utils_base\")\n", - "\n", - "def tokenize_function(examples):\n", - " with CaptureLogger(tok_logger) as cl:\n", - " output = tokenizer(examples[text_column_name])\n", - " # clm input could be much much longer than block_size\n", - " if \"Token indices sequence length is longer than the\" in cl.out:\n", - " tok_logger.warning(\n", - " \"^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\"\n", - " )\n", - " return output\n", - "\n", - "tokenized_datasets = dataset.map(\n", - " tokenize_function,\n", - " batched=True,\n", - " num_proc=data_args.preprocessing_num_workers,\n", - " remove_columns=column_names,\n", - " load_from_cache_file=not data_args.overwrite_cache,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [], - "source": [ - "if data_args.block_size is None:\n", - " block_size = tokenizer.model_max_length\n", - " if block_size > config.max_position_embeddings:\n", - " logger.warning(\n", - " f\"The tokenizer picked seems to have a very large `model_max_length` ({tokenizer.model_max_length}). \"\n", - " \"Picking 1024 instead. You can change that default value by passing --block_size xxx.\"\n", - " )\n", - " block_size = 1024\n", - "else:\n", - " if data_args.block_size > tokenizer.model_max_length:\n", - " logger.warning(\n", - " f\"The block_size passed ({data_args.block_size}) is larger than the maximum length for the model\"\n", - " f\"({tokenizer.model_max_length}). Using block_size={tokenizer.model_max_length}.\"\n", - " )\n", - " block_size = min(data_args.block_size, tokenizer.model_max_length)\n", - "\n", - "# Main data processing function that will concatenate all texts from our dataset and generate chunks of block_size.\n", - "def group_texts(examples):\n", - " # Concatenate all texts.\n", - " concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}\n", - " total_length = len(concatenated_examples[list(examples.keys())[0]])\n", - " # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can\n", - " # customize this part to your needs.\n", - " total_length = (total_length // block_size) * block_size\n", - " # Split by chunks of max_len.\n", - " result = {\n", - " k: [t[i : i + block_size] for i in range(0, total_length, block_size)]\n", - " for k, t in concatenated_examples.items()\n", - " }\n", - " result[\"labels\"] = result[\"input_ids\"].copy()\n", - " return result" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "https://symbolize.stripped_domain/r/?trace=5cd26c,7f1df8ce520f&map= \n", - "*** SIGTERM received by PID 398567 (TID 398567) on cpu 25 from PID 392558; stack trace: ***\n", - "PC: @ 0x5cd26c (unknown) (unknown)\n", - " @ 0x7f1de7392800 976 (unknown)\n", - " @ 0x7f1df8ce5210 (unknown) (unknown)\n", - "https://symbolize.stripped_domain/r/?trace=5cd26c,7f1de73927ff,7f1df8ce520f&map=2a762cd764e70bc90ae4c7f9747c08d7:7f1dda450000-7f1de76d1280 \n", - "E0705 16:13:48.265578 398567 coredump_hook.cc:250] RAW: Remote crash gathering disabled for SIGTERM.\n", - "E0705 16:13:48.276141 398567 process_state.cc:771] RAW: Raising signal 15 with default behavior\n" - ] - } - ], - "source": [ - "# Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a remainder\n", - "# for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value might be slower\n", - "# to preprocess.\n", - "#\n", - "# To speed up this part, we use multiprocessing. See the documentation of the map method for more information:\n", - "# https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map\n", - "\n", - "lm_datasets = tokenized_datasets.map(\n", - " group_texts,\n", - " batched=True,\n", - " num_proc=data_args.preprocessing_num_workers,\n", - " load_from_cache_file=not data_args.overwrite_cache,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2021-07-05 16:13:48.414578: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n", - "2021-07-05 16:13:48.414616: W tensorflow/stream_executor/cuda/cuda_driver.cc:269] failed call to cuInit: UNKNOWN ERROR (303)\n" - ] - } - ], - "source": [ - "if training_args.do_train:\n", - " if \"train\" not in tokenized_datasets:\n", - " raise ValueError(\"--do_train requires a train dataset\")\n", - " train_dataset = lm_datasets[\"train\"]\n", - " if data_args.max_train_samples is not None:\n", - " train_dataset = train_dataset.select(range(data_args.max_train_samples))\n", - "\n", - "if training_args.do_eval:\n", - " if \"validation\" not in tokenized_datasets:\n", - " raise ValueError(\"--do_eval requires a validation dataset\")\n", - " eval_dataset = lm_datasets[\"validation\"]\n", - " if data_args.max_eval_samples is not None:\n", - " eval_dataset = eval_dataset.select(range(data_args.max_eval_samples))\n", - "\n", - "# Enable tensorboard only on the master node\n", - "if has_tensorboard and jax.process_index() == 0:\n", - " summary_writer = SummaryWriter(log_dir=Path(training_args.output_dir))" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [], - "source": [ - "# Initialize our training\n", - "rng = jax.random.PRNGKey(training_args.seed)\n", - "rng, dropout_rng = jax.random.split(rng)\n", - "\n", - "# Store some constant\n", - "num_epochs = int(training_args.num_train_epochs)\n", - "train_batch_size = int(training_args.per_device_train_batch_size) * jax.device_count()\n", - "eval_batch_size = int(training_args.per_device_eval_batch_size) * jax.device_count()\n", - "steps_per_epoch = len(train_dataset) // train_batch_size\n", - "total_train_steps = steps_per_epoch * num_epochs\n", - "\n", - "# Create learning rate schedule\n", - "linear_decay_lr_schedule_fn = create_learning_rate_fn(\n", - " len(train_dataset),\n", - " train_batch_size,\n", - " training_args.num_train_epochs,\n", - " training_args.warmup_ratio,\n", - " training_args.learning_rate,\n", - ")\n", - "\n", - "# We use Optax's \"masking\" functionality to not apply weight decay\n", - "# to bias and LayerNorm scale parameters. decay_mask_fn returns a\n", - "# mask boolean with the same structure as the parameters.\n", - "# The mask is True for parameters that should be decayed.\n", - "# Note that this mask is specifically adapted for FlaxGPT2.\n", - "# For other models, one should correct the layer norm parameter naming\n", - "# accordingly.\n", - "def decay_mask_fn(params):\n", - " flat_params = traverse_util.flatten_dict(params)\n", - " flat_mask = {\n", - " path: (path[-1] != \"bias\" and path[-2:] not in [(\"ln_1\", \"scale\"), (\"ln_2\", \"scale\"), (\"ln_f\", \"scale\")])\n", - " for path in flat_params\n", - " }\n", - " return traverse_util.unflatten_dict(flat_mask)" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [], - "source": [ - "# create optimizer\n", - "if training_args.adafactor:\n", - " #TODO: figure out proper hyperparameters\n", - " adafactor = optax.adafactor(\n", - " learning_rate=linear_decay_lr_schedule_fn,\n", - " )\n", - "else:\n", - " adamw = optax.adamw(\n", - " learning_rate=linear_decay_lr_schedule_fn,\n", - " b1=training_args.adam_beta1,\n", - " b2=training_args.adam_beta2,\n", - " eps=training_args.adam_epsilon,\n", - " weight_decay=training_args.weight_decay,\n", - " mask=decay_mask_fn,\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/arto/jenv/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.\n", - " warnings.warn(\n", - "/home/arto/jenv/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.\n", - " warnings.warn(\n" - ] - } - ], - "source": [ - "# Setup train state\n", - "state = TrainState.create(apply_fn=model.__call__, params=model.params, tx=adamw, dropout_rng=dropout_rng)\n", - "\n", - "def loss_fn(logits, labels):\n", - " shift_logits = logits[..., :-1, :]\n", - " shift_labels = labels[..., 1:]\n", - " loss = optax.softmax_cross_entropy(shift_logits, onehot(shift_labels, shift_logits.shape[-1]))\n", - " return loss.mean()\n", - "\n", - "# Define gradient update step fn\n", - "def train_step(state, batch):\n", - " dropout_rng, new_dropout_rng = jax.random.split(state.dropout_rng)\n", - "\n", - " def compute_loss(params):\n", - " labels = batch.pop(\"labels\")\n", - " logits = state.apply_fn(**batch, params=params, dropout_rng=dropout_rng, train=True)[0]\n", - " loss = loss_fn(logits, labels)\n", - " return loss\n", - "\n", - " grad_fn = jax.value_and_grad(compute_loss)\n", - " loss, grad = grad_fn(state.params)\n", - " grad = jax.lax.pmean(grad, \"batch\")\n", - "\n", - " new_state = state.apply_gradients(grads=grad, dropout_rng=new_dropout_rng)\n", - "\n", - " metrics = {\"loss\": loss, \"learning_rate\": linear_decay_lr_schedule_fn(state.step)}\n", - " metrics = jax.lax.pmean(metrics, axis_name=\"batch\")\n", - "\n", - " return new_state, metrics\n", - "\n", - "# Define eval fn\n", - "def eval_step(params, batch):\n", - " labels = batch.pop(\"labels\")\n", - " logits = model(**batch, params=params, train=False)[0]\n", - " loss = loss_fn(logits, labels)\n", - "\n", - " # summarize metrics\n", - " metrics = {\"loss\": loss}\n", - " metrics = jax.lax.pmean(metrics, axis_name=\"batch\")\n", - " return metrics\n", - "\n", - "# Create parallel version of the train and eval step\n", - "p_train_step = jax.pmap(train_step, \"batch\", donate_argnums=(0,))\n", - "p_eval_step = jax.pmap(eval_step, \"batch\")\n", - "\n", - "# Replicate the train state on each device\n", - "state = state.replicate()" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [], - "source": [ - "import jax.profiler\n", - "server = jax.profiler.start_server(9999)" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:__main__:***** Running training *****\n", - "INFO:__main__: Num examples = 388109\n", - "INFO:__main__: Num Epochs = 1\n", - "INFO:__main__: Instantaneous batch size per device = 32\n", - "INFO:__main__: Total train batch size (w. parallel & distributed) = 256\n", - "INFO:__main__: Total optimization steps = 1516\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "30dcec1042fe4d289eb916a1523de78f", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Epoch ... (1/1): 0%| | 0/1 [00:00 Callable[[int], jnp.array]:\n", - " \"\"\"Returns a linear warmup, linear_decay learning rate function.\"\"\"\n", - " steps_per_epoch = train_ds_size // train_batch_size\n", - " num_train_steps = steps_per_epoch * num_train_epochs\n", - " warmup_fn = optax.linear_schedule(init_value=0.0, end_value=learning_rate, transition_steps=num_warmup_steps)\n", - " decay_fn = optax.linear_schedule(\n", - " init_value=learning_rate, end_value=0, transition_steps=num_train_steps - num_warmup_steps\n", - " )\n", - " schedule_fn = optax.join_schedules(schedules=[warmup_fn, decay_fn], boundaries=[num_warmup_steps])\n", - " return schedule_fn" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b87fb52e-7e4b-4a69-8c63-fe739331c1c5", - "metadata": {}, - "outputs": [], - "source": [ - "model_args = ModelArguments(\n", - " model_name_or_path=\"EleutherAI/gpt-neo-125M\",\n", - " dtype=\"bfloat16\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "75c30ff3-94ec-437f-af79-17ad8429d7eb", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "data_args = DataTrainingArguments(\n", - " dataset_name=\"code_search_net\", \n", - " dataset_config_name=\"python\", \n", - " block_size=1024,\n", - " max_train_samples=10000, \n", - " max_eval_samples=1000, \n", - " preprocessing_num_workers=8\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e618fc54-07d1-4e6c-bb48-7e050044a0c8", - "metadata": {}, - "outputs": [], - "source": [ - "bs = 16\n", - "training_args = TrainingArguments(\n", - " num_train_epochs=1,\n", - " output_dir=\"./tmp\", \n", - " per_device_train_batch_size=bs, \n", - " per_device_eval_batch_size=bs, \n", - " learning_rate=3e-4,\n", - " weight_decay=0.1,\n", - " do_train=True,\n", - " do_eval=True,\n", - " warmup_steps=100,\n", - " push_to_hub=False,\n", - " overwrite_output_dir=True,\n", - " report_to=None\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "20645921-b444-4e8d-b582-08af685b42d0", - "metadata": {}, - "outputs": [], - "source": [ - "def save_checkpoint(model, save_dir, state):\n", - " state = jax_utils.unreplicate(state)\n", - " print(f\"SAVING CHECKPOINT IN {save_dir}\", end=\" ... \")\n", - " model.save_pretrained(\n", - " training_args.output_dir,\n", - " params=state.params,\n", - " push_to_hub=training_args.push_to_hub,\n", - " commit_message=f\"Saving weights and logs of epoch {epoch+1}\",\n", - " )\n", - " with open(os.path.join(save_dir, \"opt_state.msgpack\"), \"wb\") as f:\n", - " f.write(to_bytes(state.opt_state))\n", - " with open(os.path.join(save_dir, \"training_state.json\"), \"w\") as f:\n", - " json.dump({\"step\": state.step.item()}, f)\n", - " print(\"DONE\")\n", - " \n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5d11fd82-435d-4afd-b1eb-9914cd465a18", - "metadata": {}, - "outputs": [], - "source": [ - "def restore_checkpoint(save_dir, state):\n", - " print(f\"RESTORING CHECKPOINT FROM {save_dir}\", end=\" ... \")\n", - " with open(os.path.join(save_dir, \"flax_model.msgpack\"), \"rb\") as f:\n", - " params = from_bytes(state.params, f.read())\n", - "\n", - " with open(os.path.join(save_dir, \"opt_state.msgpack\"), \"rb\") as f:\n", - " opt_state = from_bytes(state.opt_state, f.read())\n", - "\n", - " with open(os.path.join(save_dir, \"training_state.json\"), \"r\") as f:\n", - " training_state = json.load(f)\n", - " step = training_state[\"step\"]\n", - "\n", - " print(\"DONE\")\n", - " return params, opt_state, step" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "id": "562e0fd9-83b5-4d51-9232-df97fca4f063", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:absl:A polynomial schedule was set with a non-positive `transition_steps` value; this results in a constant schedule with value `init_value`.\n" - ] - } - ], - "source": [ - "linear_decay_lr_schedule_fn = create_learning_rate_fn(\n", - " 10000,\n", - " 128,\n", - " training_args.num_train_epochs,\n", - " training_args.warmup_steps,\n", - " training_args.learning_rate,\n", - " )\n", - "\n", - "def decay_mask_fn(params):\n", - " flat_params = traverse_util.flatten_dict(params)\n", - " flat_mask = {\n", - " path: (path[-1] != \"bias\" and path[-2:] not in [(\"ln_1\", \"scale\"), (\"ln_2\", \"scale\"), (\"ln_f\", \"scale\")])\n", - " for path in flat_params\n", - " }\n", - " return traverse_util.unflatten_dict(flat_mask)\n", - "\n", - "optimizer = optax.adamw(\n", - " learning_rate=linear_decay_lr_schedule_fn,\n", - " b1=training_args.adam_beta1,\n", - " b2=training_args.adam_beta2,\n", - " eps=training_args.adam_epsilon,\n", - " weight_decay=training_args.weight_decay,\n", - " mask=decay_mask_fn,\n", - " )\n", - "model = FlaxAutoModelForCausalLM.from_pretrained(training_args.output_dir)\n", - "rng = jax.random.PRNGKey(training_args.seed)\n", - "rng, dropout_rng = jax.random.split(rng)\n", - "state = TrainState.create(apply_fn=model.__call__, params=model.params, tx=optimizer, dropout_rng=dropout_rng)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "id": "56d2d8be-3516-4f86-b0e5-20b3a9079163", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "RESTORING CHECKPOINT FROM ./tmp ... DONE\n" - ] - } - ], - "source": [ - "params, opt_state, step = restore_checkpoint(training_args.output_dir, state)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0db5eca6-f081-4c3e-a7f1-3e080da8cc89", - "metadata": {}, - "outputs": [], - "source": [ - "if (\n", - " os.path.exists(training_args.output_dir)\n", - " and os.listdir(training_args.output_dir)\n", - " and training_args.do_train\n", - " and not training_args.overwrite_output_dir\n", - "):\n", - " raise ValueError(\n", - " f\"Output directory ({training_args.output_dir}) already exists and is not empty.\"\n", - " \"Use --overwrite_output_dir to overcome.\"\n", - " )\n", - "\n", - "# Make one log on every process with the configuration for debugging.\n", - "logging.basicConfig(\n", - " format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n", - " datefmt=\"%m/%d/%Y %H:%M:%S\",\n", - " level=logging.INFO,\n", - ")\n", - "# Setup logging, we only want one process per machine to log things on the screen.\n", - "logger.setLevel(logging.INFO if jax.process_index() == 0 else logging.ERROR)\n", - "if jax.process_index() == 0:\n", - " datasets.utils.logging.set_verbosity_warning()\n", - " transformers.utils.logging.set_verbosity_info()\n", - "else:\n", - " datasets.utils.logging.set_verbosity_error()\n", - " transformers.utils.logging.set_verbosity_error()\n", - "\n", - "# Set the verbosity to info of the Transformers logger (on main process only):\n", - "logger.info(f\"Training/evaluation parameters {training_args}\")\n", - "\n", - "# Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)\n", - "# or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/\n", - "# (the dataset will be downloaded automatically from the datasets Hub).\n", - "#\n", - "# For CSV/JSON files, this script will use the column called 'text' or the first column if no column called\n", - "# 'text' is found. You can easily tweak this behavior (see below).\n", - "#\n", - "# In distributed training, the load_dataset function guarantees that only one local process can concurrently\n", - "# download the dataset.\n", - "if data_args.dataset_name is not None:\n", - " # Downloading and loading a dataset from the hub.\n", - " dataset = load_dataset(\n", - " data_args.dataset_name, data_args.dataset_config_name, cache_dir=model_args.cache_dir, keep_in_memory=False\n", - " )\n", - "\n", - " if \"validation\" not in dataset.keys():\n", - " dataset[\"validation\"] = load_dataset(\n", - " data_args.dataset_name,\n", - " data_args.dataset_config_name,\n", - " split=f\"train[:{data_args.validation_split_percentage}%]\",\n", - " cache_dir=model_args.cache_dir,\n", - " )\n", - " dataset[\"train\"] = load_dataset(\n", - " data_args.dataset_name,\n", - " data_args.dataset_config_name,\n", - " split=f\"train[{data_args.validation_split_percentage}%:]\",\n", - " cache_dir=model_args.cache_dir,\n", - " )\n", - "else:\n", - " data_files = {}\n", - " if data_args.train_file is not None:\n", - " data_files[\"train\"] = data_args.train_file\n", - " if data_args.validation_file is not None:\n", - " data_files[\"validation\"] = data_args.validation_file\n", - " extension = data_args.train_file.split(\".\")[-1]\n", - " if extension == \"txt\":\n", - " extension = \"text\"\n", - " dataset = load_dataset(extension, data_files=data_files, cache_dir=model_args.cache_dir)\n", - "# See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at\n", - "# https://huggingface.co/docs/datasets/loading_datasets.html.\n", - "\n", - "# Load pretrained model and tokenizer\n", - "\n", - "# Distributed training:\n", - "# The .from_pretrained methods guarantee that only one local process can concurrently\n", - "# download model & vocab.\n", - "if model_args.config_name:\n", - " config = AutoConfig.from_pretrained(model_args.config_name, cache_dir=model_args.cache_dir)\n", - "elif model_args.model_name_or_path:\n", - " config = AutoConfig.from_pretrained(model_args.model_name_or_path, cache_dir=model_args.cache_dir)\n", - "else:\n", - " config = CONFIG_MAPPING[model_args.model_type]()\n", - " logger.warning(\"You are instantiating a new config instance from scratch.\")\n", - "\n", - "if model_args.tokenizer_name:\n", - " tokenizer = AutoTokenizer.from_pretrained(\n", - " model_args.tokenizer_name, cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer\n", - " )\n", - "elif model_args.model_name_or_path:\n", - " tokenizer = AutoTokenizer.from_pretrained(\n", - " model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer\n", - " )\n", - "else:\n", - " raise ValueError(\n", - " \"You are instantiating a new tokenizer from scratch. This is not supported by this script.\"\n", - " \"You can do it from another script, save it, and load it from here, using --tokenizer_name.\"\n", - " )\n", - "\n", - "if model_args.model_name_or_path:\n", - " model = FlaxAutoModelForCausalLM.from_pretrained(\n", - " model_args.model_name_or_path, config=config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype)\n", - " )\n", - "else:\n", - " model = FlaxAutoModelForCausalLM.from_config(\n", - " config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype)\n", - " )\n", - "\n", - "# Preprocessing the datasets.\n", - "# First we tokenize all the texts.\n", - "if training_args.do_train:\n", - " column_names = dataset[\"train\"].column_names\n", - "else:\n", - " column_names = dataset[\"validation\"].column_names\n", - "text_column_name = data_args.text_column_name if data_args.text_column_name in column_names else column_names[0]\n", - "\n", - "# since this will be pickled to avoid _LazyModule error in Hasher force logger loading before tokenize_function\n", - "tok_logger = transformers.utils.logging.get_logger(\"transformers.tokenization_utils_base\")\n", - "\n", - "def tokenize_function(examples):\n", - " with CaptureLogger(tok_logger) as cl:\n", - " output = tokenizer(examples[text_column_name])\n", - " # clm input could be much much longer than block_size\n", - " if \"Token indices sequence length is longer than the\" in cl.out:\n", - " tok_logger.warning(\n", - " \"^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\"\n", - " )\n", - " return output\n", - "\n", - "tokenized_datasets = dataset.map(\n", - " tokenize_function,\n", - " batched=True,\n", - " num_proc=data_args.preprocessing_num_workers,\n", - " remove_columns=column_names,\n", - " load_from_cache_file=not data_args.overwrite_cache,\n", - ")\n", - "\n", - "if data_args.block_size is None:\n", - " block_size = tokenizer.model_max_length\n", - " if block_size > config.max_position_embeddings:\n", - " logger.warning(\n", - " f\"The tokenizer picked seems to have a very large `model_max_length` ({tokenizer.model_max_length}). \"\n", - " \"Picking 1024 instead. You can change that default value by passing --block_size xxx.\"\n", - " )\n", - " block_size = 1024\n", - "else:\n", - " if data_args.block_size > tokenizer.model_max_length:\n", - " logger.warning(\n", - " f\"The block_size passed ({data_args.block_size}) is larger than the maximum length for the model\"\n", - " f\"({tokenizer.model_max_length}). Using block_size={tokenizer.model_max_length}.\"\n", - " )\n", - " block_size = min(data_args.block_size, tokenizer.model_max_length)\n", - "\n", - "# Main data processing function that will concatenate all texts from our dataset and generate chunks of block_size.\n", - "def group_texts(examples):\n", - " # Concatenate all texts.\n", - " concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}\n", - " total_length = len(concatenated_examples[list(examples.keys())[0]])\n", - " # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can\n", - " # customize this part to your needs.\n", - " total_length = (total_length // block_size) * block_size\n", - " # Split by chunks of max_len.\n", - " result = {\n", - " k: [t[i : i + block_size] for i in range(0, total_length, block_size)]\n", - " for k, t in concatenated_examples.items()\n", - " }\n", - " result[\"labels\"] = result[\"input_ids\"].copy()\n", - " return result\n", - "\n", - "# Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a remainder\n", - "# for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value might be slower\n", - "# to preprocess.\n", - "#\n", - "# To speed up this part, we use multiprocessing. See the documentation of the map method for more information:\n", - "# https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map\n", - "\n", - "lm_datasets = tokenized_datasets.map(\n", - " group_texts,\n", - " batched=True,\n", - " num_proc=data_args.preprocessing_num_workers,\n", - " load_from_cache_file=not data_args.overwrite_cache,\n", - ")\n", - "\n", - "if training_args.do_train:\n", - " if \"train\" not in tokenized_datasets:\n", - " raise ValueError(\"--do_train requires a train dataset\")\n", - " train_dataset = lm_datasets[\"train\"]\n", - " if data_args.max_train_samples is not None:\n", - " train_dataset = train_dataset.select(range(data_args.max_train_samples))\n", - "\n", - "if training_args.do_eval:\n", - " if \"validation\" not in tokenized_datasets:\n", - " raise ValueError(\"--do_eval requires a validation dataset\")\n", - " eval_dataset = lm_datasets[\"validation\"]\n", - " if data_args.max_eval_samples is not None:\n", - " eval_dataset = eval_dataset.select(range(data_args.max_eval_samples))\n", - "\n", - "# Enable tensorboard only on the master node\n", - "has_tensorboard = is_tensorboard_available()\n", - "if has_tensorboard and jax.process_index() == 0:\n", - " try:\n", - " from flax.metrics.tensorboard import SummaryWriter\n", - "\n", - " summary_writer = SummaryWriter(log_dir=Path(training_args.output_dir))\n", - " except ImportError as ie:\n", - " has_tensorboard = False\n", - " logger.warning(\n", - " f\"Unable to display metrics through TensorBoard because some package are not installed: {ie}\"\n", - " )\n", - "else:\n", - " logger.warning(\n", - " \"Unable to display metrics through TensorBoard because the package is not installed: \"\n", - " \"Please run pip install tensorboard to enable.\"\n", - " )\n", - "\n", - "# Initialize our training\n", - "rng = jax.random.PRNGKey(training_args.seed)\n", - "rng, dropout_rng = jax.random.split(rng)\n", - "\n", - "# Store some constant\n", - "num_epochs = int(training_args.num_train_epochs)\n", - "train_batch_size = int(training_args.per_device_train_batch_size) * jax.device_count()\n", - "eval_batch_size = int(training_args.per_device_eval_batch_size) * jax.device_count()\n", - "steps_per_epoch = len(train_dataset) // train_batch_size\n", - "total_train_steps = steps_per_epoch * num_epochs\n", - "\n", - "# Create learning rate schedule\n", - "linear_decay_lr_schedule_fn = create_learning_rate_fn(\n", - " len(train_dataset),\n", - " train_batch_size,\n", - " training_args.num_train_epochs,\n", - " training_args.warmup_steps,\n", - " training_args.learning_rate,\n", - ")\n", - "\n", - "# We use Optax's \"masking\" functionality to not apply weight decay\n", - "# to bias and LayerNorm scale parameters. decay_mask_fn returns a\n", - "# mask boolean with the same structure as the parameters.\n", - "# The mask is True for parameters that should be decayed.\n", - "# Note that this mask is specifically adapted for FlaxGPT2.\n", - "# For other models, one should correct the layer norm parameter naming\n", - "# accordingly.\n", - "def decay_mask_fn(params):\n", - " flat_params = traverse_util.flatten_dict(params)\n", - " flat_mask = {\n", - " path: (path[-1] != \"bias\" and path[-2:] not in [(\"ln_1\", \"scale\"), (\"ln_2\", \"scale\"), (\"ln_f\", \"scale\")])\n", - " for path in flat_params\n", - " }\n", - " return traverse_util.unflatten_dict(flat_mask)\n", - "\n", - "# create adam optimizer\n", - "adamw = optax.adamw(\n", - " learning_rate=linear_decay_lr_schedule_fn,\n", - " b1=training_args.adam_beta1,\n", - " b2=training_args.adam_beta2,\n", - " eps=training_args.adam_epsilon,\n", - " weight_decay=training_args.weight_decay,\n", - " mask=decay_mask_fn,\n", - ")\n", - "\n", - "# Setup train state\n", - "state = TrainState.create(apply_fn=model.__call__, params=model.params, tx=adamw, dropout_rng=dropout_rng)\n", - "\n", - "def loss_fn(logits, labels):\n", - " shift_logits = logits[..., :-1, :]\n", - " shift_labels = labels[..., 1:]\n", - " loss = optax.softmax_cross_entropy(shift_logits, onehot(shift_labels, shift_logits.shape[-1]))\n", - " return loss.mean()\n", - "\n", - "# Define gradient update step fn\n", - "def train_step(state, batch):\n", - " dropout_rng, new_dropout_rng = jax.random.split(state.dropout_rng)\n", - "\n", - " def compute_loss(params):\n", - " labels = batch.pop(\"labels\")\n", - " logits = state.apply_fn(**batch, params=params, dropout_rng=dropout_rng, train=True)[0]\n", - " loss = loss_fn(logits, labels)\n", - " return loss\n", - "\n", - " grad_fn = jax.value_and_grad(compute_loss)\n", - " loss, grad = grad_fn(state.params)\n", - " grad = jax.lax.pmean(grad, \"batch\")\n", - "\n", - " new_state = state.apply_gradients(grads=grad, dropout_rng=new_dropout_rng)\n", - "\n", - " metrics = {\"loss\": loss, \"learning_rate\": linear_decay_lr_schedule_fn(state.step)}\n", - " metrics = jax.lax.pmean(metrics, axis_name=\"batch\")\n", - "\n", - " return new_state, metrics\n", - "\n", - "# Define eval fn\n", - "def eval_step(params, batch):\n", - " labels = batch.pop(\"labels\")\n", - " logits = model(**batch, params=params, train=False)[0]\n", - " loss = loss_fn(logits, labels)\n", - "\n", - " # summarize metrics\n", - " metrics = {\"loss\": loss}\n", - " metrics = jax.lax.pmean(metrics, axis_name=\"batch\")\n", - " return metrics\n", - "\n", - "# Create parallel version of the train and eval step\n", - "p_train_step = jax.pmap(train_step, \"batch\", donate_argnums=(0,))\n", - "p_eval_step = jax.pmap(eval_step, \"batch\")\n", - "\n", - "# Replicate the train state on each device\n", - "state = state.replicate()\n", - "\n", - "logger.info(\"***** Running training *****\")\n", - "logger.info(f\" Num examples = {len(train_dataset)}\")\n", - "logger.info(f\" Num Epochs = {num_epochs}\")\n", - "logger.info(f\" Instantaneous batch size per device = {training_args.per_device_train_batch_size}\")\n", - "logger.info(f\" Total train batch size (w. parallel & distributed) = {train_batch_size}\")\n", - "logger.info(f\" Total optimization steps = {total_train_steps}\")\n", - "\n", - "train_time = 0\n", - "train_metrics = []\n", - "epochs = tqdm(range(num_epochs), desc=f\"Epoch ... (1/{num_epochs})\", position=0)\n", - "for epoch in epochs:\n", - " # ======================== Training ================================\n", - " train_start = time.time()\n", - "\n", - " # Create sampling rng\n", - " rng, input_rng = jax.random.split(rng)\n", - "\n", - " # Generate an epoch by shuffling sampling indices from the train dataset\n", - " train_loader = data_loader(input_rng, train_dataset, train_batch_size, shuffle=True)\n", - " steps_per_epoch = len(train_dataset) // train_batch_size\n", - " # train\n", - " for step in tqdm(range(steps_per_epoch), desc=\"Training...\", position=1, leave=False):\n", - " batch = next(train_loader)\n", - " state, train_metric = p_train_step(state, batch)\n", - " train_metrics.append(train_metric)\n", - "\n", - " cur_step = epoch * (len(train_dataset) // train_batch_size) + step\n", - "\n", - " if cur_step % training_args.logging_steps == 0 and cur_step > 0:\n", - " # Save metrics\n", - " train_metric = unreplicate(train_metric)\n", - " train_time += time.time() - train_start\n", - " if has_tensorboard and jax.process_index() == 0:\n", - " write_train_metric(summary_writer, train_metrics, train_time, cur_step)\n", - "\n", - " epochs.write(\n", - " f\"Step... ({cur_step} | Loss: {train_metric['loss'].mean()}, Learning Rate: {train_metric['learning_rate'].mean()})\"\n", - " )\n", - "\n", - " train_metrics = []\n", - "\n", - " # ======================== Evaluating ==============================\n", - " eval_metrics = []\n", - " eval_loader = data_loader(input_rng, eval_dataset, eval_batch_size)\n", - " eval_steps = len(eval_dataset) // eval_batch_size\n", - " for _ in tqdm(range(eval_steps), desc=\"Evaluating...\", position=2, leave=False):\n", - " # Model forward\n", - " batch = next(eval_loader)\n", - " metrics = p_eval_step(state.params, batch)\n", - " eval_metrics.append(metrics)\n", - "\n", - " # normalize eval metrics\n", - " eval_metrics = get_metrics(eval_metrics)\n", - "\n", - " eval_metrics = jax.tree_map(jnp.mean, eval_metrics)\n", - "\n", - " try:\n", - " eval_metrics[\"perplexity\"] = math.exp(eval_metrics[\"loss\"])\n", - " except OverflowError:\n", - " eval_metrics[\"perplexity\"] = float(\"inf\")\n", - "\n", - " # Print metrics and update progress bar\n", - " desc = f\"Epoch... ({epoch + 1}/{num_epochs} | Eval Loss: {eval_metrics['loss']} | Eval Perplexity: {eval_metrics['perplexity']})\"\n", - " epochs.write(desc)\n", - " epochs.desc = desc\n", - "\n", - " # Save metrics\n", - " if has_tensorboard and jax.process_index() == 0:\n", - " cur_step = epoch * (len(train_dataset) // train_batch_size)\n", - " write_eval_metric(summary_writer, eval_metrics, cur_step)\n", - "\n", - " # save checkpoint after each epoch and push checkpoint to the hub\n", - " if jax.process_index() == 0:\n", - " save_checkpoint(model, training_args.output_dir, state)\n", - " \n", - "\n", - "\n", - "# if __name__ == \"__main__\":\n", - "# main()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f39555f2-ccf7-458c-8756-be1ff330229b", - "metadata": {}, - "outputs": [], - "source": [ - "save_checkpoint(model, training_args.output_dir, state)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/flax-gpt-neo-clm.ipynb b/flax-gpt-neo-clm.ipynb deleted file mode 100644 index c263ca5..0000000 --- a/flax-gpt-neo-clm.ipynb +++ /dev/null @@ -1,6626 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "uGYl4nCPKyZi" - }, - "source": [ - "# Fine-tuning GPTNeo with 🤗 Transformers and **Flax/JAX** on TPU\n", - "\n", - "In this notebook, we will see how to pretrain one of the [🤗 Transformers](https://github.com/huggingface/transformers) models on TPU using [**Flax**](https://flax.readthedocs.io/en/latest/index.html). \n", - "\n", - "GPTNeo causal language modeling objective will be used for pre-training here.\n", - "\n", - "As can be seen on [this benchmark](https://github.com/huggingface/transformers/tree/master/examples/flax/language-modeling#runtime-evaluation) using Flax/JAX on GPU/TPU is often much faster and can also be considerably cheaper than using PyTorch on GPU/TPU.\n", - "\n", - "[**Flax**](https://flax.readthedocs.io/en/latest/index.html) is a high-performance neural network library designed for flexibility built on top of JAX (see below). It aims to provide users with full control of their training code and is carefully designed to work well with JAX transformations such as `grad` and `pmap` (see the [Flax philosophy](https://flax.readthedocs.io/en/latest/philosophy.html)). For an introduction to Flax see the [Flax Basic Colab](https://flax.readthedocs.io/en/latest/notebooks/flax_basics.html) or the list of curated [Flax examples](https://flax.readthedocs.io/en/latest/examples.html).\n", - "\n", - "[**JAX**](https://jax.readthedocs.io/en/latest/index.html) is Autograd and XLA, brought together for high-performance numerical computing and machine learning research. It provides composable transformations of Python+NumPy programs: differentiate, vectorize, parallelize, Just-In-Time compile to GPU/TPU, and more. A great place for getting started with JAX is the [JAX 101 Tutorial](https://jax.readthedocs.io/en/latest/jax-101/index.html)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "PwDAzFXQMd46" - }, - "source": [ - "If you're opening this Notebook on colab, you will probably need to install 🤗 Transformers, 🤗 Datasets, 🤗 Tokenizers as well as [Flax](https://github.com/google/flax.git) and [Optax](https://github.com/deepmind/optax). Optax is a gradient processing and optimization library for JAX, and is the optimizer library\n", - "recommended by Flax." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "If_SYBvU5V6u" - }, - "source": [ - "If everything is set up correctly, the following command should return a list of 8 TPU devices." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!export HF_HOME=\"/home/shared/.cache/hf/\"" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "3R5MP7PAbV7V", - "outputId": "9cf6b9a4-7b9c-4029-d938-dcf9a3ebb4c4" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "[TpuDevice(id=0, process_index=0, coords=(0,0,0), core_on_chip=0),\n", - " TpuDevice(id=1, process_index=0, coords=(0,0,0), core_on_chip=1),\n", - " TpuDevice(id=2, process_index=0, coords=(1,0,0), core_on_chip=0),\n", - " TpuDevice(id=3, process_index=0, coords=(1,0,0), core_on_chip=1),\n", - " TpuDevice(id=4, process_index=0, coords=(0,1,0), core_on_chip=0),\n", - " TpuDevice(id=5, process_index=0, coords=(0,1,0), core_on_chip=1),\n", - " TpuDevice(id=6, process_index=0, coords=(1,1,0), core_on_chip=0),\n", - " TpuDevice(id=7, process_index=0, coords=(1,1,0), core_on_chip=1)]" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import jax\n", - "jax.local_devices()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "vehXZCipMa1V" - }, - "source": [ - "In this notebook, we will pre-train an [autoregressive model](https://huggingface.co/transformers/model_summary.html#autoregressive-models) on one of the languages of the OSCAR corpus. [OSCAR](https://oscar-corpus.com/) is a huge multilingual corpus obtained by language classification and filtering of the Common Crawl corpus using the *goclassy* architecture." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "iz8HrV8JPHn0" - }, - "source": [ - "Let's first select the language that our model should learn.\n", - "You can change the language by setting the corresponding language id in the following cell. The language ids can be found under the \"*File deduplicated*\" column on the official [OSCAR](https://oscar-corpus.com/) website.\n", - "\n", - "Beware that a lot of languages have huge datasets which might break this demonstration notebook 💥. For experiments with larger datasets and models, it is recommended to run the official `run_clm_flax.py` script offline that can be found [here](https://github.com/huggingface/transformers/tree/master/examples/flax/language-modeling#masked-language-modeling).\n", - "\n", - "Here we select `is` for Icelandic 🇮🇸." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "ii9XwLsmiY-E" - }, - "outputs": [], - "source": [ - "ds_name = \"code_search_net\"\n", - "language = \"python\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jVtv6T0oSjNq" - }, - "source": [ - "Next, we select the model architecture to be trained from scratch.\n", - "Here we choose [**`distilgpt2`**](https://huggingface.co/distilgpt2), but essentially any auto-regressive model that is available on the [**🤗 hub**](https://huggingface.co/models?filter=masked-lm,jax) in JAX/Flax can be used. " - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "from ipywidgets import Dropdown" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "bc69b75cca024772ab08e70c811dca94", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Dropdown(options=('EleutherAI/gpt-neo-125M', 'EleutherAI/gpt-neo-1.3B', 'EleutherAI/gpt-neo-2.7B'), value='Ele…" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "ckpt_selector = Dropdown(options = [\"EleutherAI/gpt-neo-125M\", \"EleutherAI/gpt-neo-1.3B\", \"EleutherAI/gpt-neo-2.7B\"])\n", - "ckpt_selector" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "id": "Sj1mJNJa6PPS" - }, - "outputs": [], - "source": [ - "model_ckpt = ckpt_selector.value" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "j-tf_3Ch55_9" - }, - "source": [ - "## 1. Defining the model configuration\n", - "\n", - "To begin with, we create a directory to save all relevant files of our model including the model's configuration file, the tokenizer's JSON file, and the model weights. We call the directory `\"distilgpt2-base-pretrained-is\"`:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "id": "1dwuSvQxeM8-" - }, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "model_dir = Path.home()/f\"{model_ckpt.split('/')[1]}-code-clippy\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "qGENnc6LeRFL" - }, - "source": [ - "and create it:" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "oWQD8IA9eAFY" - }, - "source": [ - "Next, we'll download the model configuration:" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "id": "DO1SwHdi55en" - }, - "outputs": [], - "source": [ - "# from transformers import AutoConfig\n", - "\n", - "# config = AutoConfig.from_pretrained(model_config)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "3exPFi-keYlT" - }, - "source": [ - " and save it to the directory:" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "id": "Vip8WKEp6b6Y" - }, - "outputs": [], - "source": [ - "# config.save_pretrained(f\"{model_dir}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "id": "kJKw0tqOcDu6" - }, - "outputs": [], - "source": [ - "from datasets import load_dataset\n", - "from transformers import AutoTokenizer" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "3cQXZ1p5XHtP" - }, - "source": [ - "We will store our tokenizer files and model files in a directory, called `model_dir`. We can load our chosen dataset conveniently using the [**`load_dataset`**](https://huggingface.co/docs/datasets/package_reference/loading_methods.html?highlight=load_dataset#datasets.load_dataset) function." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "5oUW__q-4If7", - "outputId": "4e5f1bd9-b6c1-42fe-ea21-c00b1c4ff47a" - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:datasets.builder:Reusing dataset code_search_net (/home/arto/.cache/huggingface/datasets/code_search_net/python/1.0.0/80a244ab541c6b2125350b764dc5c2b715f65f00de7a56107a28915fac173a27)\n" - ] - } - ], - "source": [ - "dataset = load_dataset(ds_name, language)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "id": "OCs_CQFt4WK_" - }, - "outputs": [], - "source": [ - "tokenizer = AutoTokenizer.from_pretrained(model_ckpt)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "4hD8d1_P5huo" - }, - "source": [ - "## 3. Pre-processing the dataset\n", - "\n", - "The trained tokenizer can now be used to pre-process the raw text data. \n", - "GPT2 was trained to generate tokens up to `1024` tokens, see paper [here](https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf).\n", - "However, since the required memory of Transformer models scales quadratically with the sequence length, we cap the maximum input length at 512 here. The raw text data is pre-processed accordingly." - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": { - "id": "uDhqWoF-MAGv" - }, - "outputs": [], - "source": [ - "max_seq_length = 128 #512" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(20000, 2000)" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "len(dataset['train']), len(dataset['validation'])" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": { - "id": "aoXFHjEtwXWt" - }, - "outputs": [], - "source": [ - "# these cells should be commented out to run on full dataset\n", - "dataset[\"train\"] = dataset[\"train\"].select(range(20000))\n", - "dataset[\"validation\"] = dataset[\"validation\"].select(range(2000))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "hYmElz46k7_E" - }, - "source": [ - "Next, we load the previously trained `ByteLevelBPETokenizer` tokenizer to pre-process the raw text data:" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": { - "id": "wcpWIxX8dIAO" - }, - "outputs": [], - "source": [ - "def tokenize_function(examples):\n", - " return tokenizer(examples[\"func_code_string\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "lco7GkZ8nF-a" - }, - "source": [ - "and apply the tokenization function to every text sample via the convenient `map(...)` function of Datasets. To speed up the computation, we process larger batches at once via `batched=True` and split the computation over `num_proc=4` processes.\n", - "\n", - "**Note**: Running this command on the whole dataset might take up to 10 minutes ☕." - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "h6cjpFO2dTYC", - "outputId": "5b57b7aa-79f5-4780-95be-eddc79760f3b" - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Token indices sequence length is longer than the specified maximum sequence length for this model (2090 > 2048). Running this sequence through the model will result in indexing errors\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (2219 > 2048). Running this sequence through the model will result in indexing errors\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (4716 > 2048). Running this sequence through the model will result in indexing errors\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (2795 > 2048). Running this sequence through the model will result in indexing errors\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (2736 > 2048). Running this sequence through the model will result in indexing errors\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (2943 > 2048). Running this sequence through the model will result in indexing errors\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (3776 > 2048). Running this sequence through the model will result in indexing errors\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (3817 > 2048). Running this sequence through the model will result in indexing errors\n", - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /home/arto/.cache/huggingface/datasets/code_search_net/python/1.0.0/80a244ab541c6b2125350b764dc5c2b715f65f00de7a56107a28915fac173a27/cache-7b66142abafedf08.arrow\n", - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /home/arto/.cache/huggingface/datasets/code_search_net/python/1.0.0/80a244ab541c6b2125350b764dc5c2b715f65f00de7a56107a28915fac173a27/cache-a010a7cca7e5eda5.arrow\n", - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /home/arto/.cache/huggingface/datasets/code_search_net/python/1.0.0/80a244ab541c6b2125350b764dc5c2b715f65f00de7a56107a28915fac173a27/cache-e31ef47e93e1b72f.arrow\n", - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /home/arto/.cache/huggingface/datasets/code_search_net/python/1.0.0/80a244ab541c6b2125350b764dc5c2b715f65f00de7a56107a28915fac173a27/cache-a0ccaa62536849b2.arrow\n", - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /home/arto/.cache/huggingface/datasets/code_search_net/python/1.0.0/80a244ab541c6b2125350b764dc5c2b715f65f00de7a56107a28915fac173a27/cache-78eeda3c362d3d04.arrow\n", - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /home/arto/.cache/huggingface/datasets/code_search_net/python/1.0.0/80a244ab541c6b2125350b764dc5c2b715f65f00de7a56107a28915fac173a27/cache-ef7a0c77b4162772.arrow\n", - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /home/arto/.cache/huggingface/datasets/code_search_net/python/1.0.0/80a244ab541c6b2125350b764dc5c2b715f65f00de7a56107a28915fac173a27/cache-94163ac3c687abcb.arrow\n", - "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /home/arto/.cache/huggingface/datasets/code_search_net/python/1.0.0/80a244ab541c6b2125350b764dc5c2b715f65f00de7a56107a28915fac173a27/cache-a707561ad73af264.arrow\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (5852 > 2048). Running this sequence through the model will result in indexing errors\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (3702 > 2048). Running this sequence through the model will result in indexing errors\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (2478 > 2048). Running this sequence through the model will result in indexing errors\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (5691 > 2048). Running this sequence through the model will result in indexing errors\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (4533 > 2048). Running this sequence through the model will result in indexing errors\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (2182 > 2048). Running this sequence through the model will result in indexing errors\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (2240 > 2048). Running this sequence through the model will result in indexing errors\n", - "Token indices sequence length is longer than the specified maximum sequence length for this model (2509 > 2048). Running this sequence through the model will result in indexing errors\n" - ] - } - ], - "source": [ - "tokenized_datasets = dataset.map(tokenize_function, batched=True, num_proc=8, remove_columns=dataset[\"train\"].column_names)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "6_E0jsY9onEf" - }, - "source": [ - "The model can process the training data most efficiently if all data samples are of the same length. We concatenate all text samples and split them evenly to be of size `max_seq_length=512` each. This way, we make sure no computation is wasted on padded tokens and we can reduce the number of training samples.\n", - "Causal Language modeling simply consists of predicting the next token which means that the labels are essentially the inputs just shifted to the left. Thus, we copy the `input_ids` tensor and set it to `labels`.\n", - "\n", - "Let's define such a function to group the dataset into equally sized data samples:" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": { - "id": "HO_neGynddat" - }, - "outputs": [], - "source": [ - "def group_texts(examples):\n", - " concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}\n", - " total_length = len(concatenated_examples[list(examples.keys())[0]])\n", - " total_length = (total_length // max_seq_length) * max_seq_length\n", - " result = {\n", - " k: [t[i : i + max_seq_length] for i in range(0, total_length, max_seq_length)]\n", - " for k, t in concatenated_examples.items()\n", - " }\n", - " result[\"labels\"] = result[\"input_ids\"].copy()\n", - " return result" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "CR46Vvpwr6e5" - }, - "source": [ - "We pass `group_texts` to the `map(...)` function and set `batched=True` to make sure that the function is applied to a large batch of data samples. \n", - "\n", - "**Note**: Running this function on the whole dataset might take up to 50 minutes 🕒." - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "UmzNAUVediDa", - "outputId": "00f5fd1c-cb16-4539-f24e-a78d7164ff85" - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "https://symbolize.stripped_domain/r/?trace=56a592,7f3f9d4bc20f,955ddf,3a0fc0f&map= \n", - "*** SIGTERM received by PID 361924 (TID 361924) on cpu 0 from PID 359912; stack trace: ***\n", - "PC: @ 0x56a592 (unknown) _PyEval_EvalFrameDefault\n", - " @ 0x7f3f8bb69800 976 (unknown)\n", - " @ 0x7f3f9d4bc210 (unknown) (unknown)\n", - " @ 0x955de0 (unknown) (unknown)\n", - " @ 0x3a0fc10 (unknown) (unknown)\n", - "https://symbolize.stripped_domain/r/?trace=56a592,7f3f8bb697ff,7f3f9d4bc20f,955ddf,3a0fc0f&map=2a762cd764e70bc90ae4c7f9747c08d7:7f3f7ec27000-7f3f8bea8280 \n", - "E0705 13:31:05.285333 361924 coredump_hook.cc:250] RAW: Remote crash gathering disabled for SIGTERM.\n", - "E0705 13:31:05.301524 361924 process_state.cc:771] RAW: Raising signal 15 with default behavior\n" - ] - } - ], - "source": [ - "tokenized_datasets = tokenized_datasets.map(group_texts, batched=True, num_proc=8)" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "], axis=0)\n", - " d_z = np.diagonal(D)[2:][0]\n", - "\n", - " cond1 = np.all(d_xy < allowed_xy_movement)\n", - " cond2 = np.all([d_z[i] < allowed_z_movement[i]\n", - " for i in range(len(a))])\n", - "\n", - " \n" - ] - } - ], - "source": [ - "import random\n", - "\n", - "id = random.randint(0, len(tokenized_datasets['train'])-1)\n", - "\n", - "example = tokenized_datasets['train'][id]\n", - "print(tokenizer.decode(example['input_ids']))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jid2JqXOsVfR" - }, - "source": [ - "Awesome, the data is now fully pre-processed and ready to be used for training 😎." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ZRvfr609LzWu" - }, - "source": [ - "## 4. Pre-Training the model\n", - "\n", - "Now we will see how the power of Google's tensor processing unit (TPU) can be leveraged with Flax/JAX for the compute-intensive pre-training of language models.\n", - "\n", - "We need to import `jax`, `flax`, `optax`, `numpy` to define our training loop. Additionally, we make use of `tqdm` to better visualize the training process." - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": { - "id": "5qOhue4Xm1TO" - }, - "outputs": [], - "source": [ - "import jax\n", - "import optax\n", - "import flax\n", - "import jax.numpy as jnp\n", - "import math\n", - "\n", - "from flax.training import train_state\n", - "from flax.training.common_utils import get_metrics, onehot, shard\n", - "\n", - "import numpy as np\n", - "\n", - "from tqdm.notebook import tqdm" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "_MGleTRG6Vor" - }, - "source": [ - "At first, we define all relevant hyper-parameters for pretraining in this notebook:\n", - "\n", - "- Each TPU will process a batch size of `16`\n", - "- The model is trained for `10` epochs\n", - "- The learning rate starts at `3e-4` and is successfully linearly decayed with each training step\n", - "- To reproduce the training run, a random seed is set to `0`.\n", - "\n", - "We can deduce the total batch size over all devices as well as the total number of training steps accordingly." - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": { - "id": "y8lsJQy8liud" - }, - "outputs": [], - "source": [ - "per_device_batch_size = 1\n", - "num_train_epochs = 1\n", - "training_seed = 0\n", - "learning_rate = 3e-4\n", - "\n", - "total_batch_size = per_device_batch_size * jax.device_count()\n", - "num_train_steps = len(tokenized_datasets[\"train\"]) // total_batch_size * num_train_epochs" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "FB9bRDBq5j3r" - }, - "source": [ - "In the [official GPT2 paper](https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf) a batch size of 512 is used.\n", - "\n", - "Here, we use a batch size of `8 * 16 = 128` due to the TPU memory constraints of this notebook. When running this script locally on a TPUv3-8, one can easily use batch sizes of up to `8 * 64 = 512`." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "i0Tylp115u1r" - }, - "source": [ - "Now we randomly initialized a `distilgpt2` model according to its configuration. To save memory and improve speed, we initialize the weights directly in `bfloat16` by setting `dtype=jnp.dtype(\"bfloat16\")`." - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": { - "id": "aVr9TCzfacLN" - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "tcmalloc: large alloc 5262319616 bytes == 0x90a30000 @ 0x7f3f9d686680 0x7f3f9d6a7824 0x5f7b11 0x648631 0x5c38e6 0x4f30e6 0x64ee88 0x505653 0x56acb6 0x568d9a 0x5f5b33 0x50b7f8 0x5f2702 0x56c332 0x568d9a 0x50b868 0x56bc9b 0x568d9a 0x68cdc7 0x5ff5d4 0x5c3cb0 0x56aadf 0x501148 0x56c422 0x501148 0x56c422 0x501148 0x504d56 0x56acb6 0x5f5956 0x56aadf\n" - ] - } - ], - "source": [ - "from transformers import FlaxAutoModelForCausalLM\n", - "\n", - "model = FlaxAutoModelForCausalLM.from_pretrained(model_ckpt, dtype=jnp.dtype(\"bfloat16\"))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "sMS_QkT76Lgk" - }, - "source": [ - "Next, we define the learning rate schedule. A simple and effective learning rate schedule is the linear decay with warmup (click [here](https://huggingface.co/transformers/main_classes/optimizer_schedules.html#transformers.get_linear_schedule_with_warmup) for more information). For simplicity, we set the number of warmup steps simply to 0 here. The schedule is then fully defined by the number of training steps and the learning rate.\n", - "\n", - "It is recommended to use the [**optax**](https://github.com/deepmind/optax) library for training utilities, *e.g.* learning rate schedules and optimizers.\n", - "\n", - "To see how to define a learning rate schedule with warmup, please take a look at the [official Flax CLM pre-training script](https://github.com/huggingface/transformers/blob/master/examples/flax/language-modeling/run_clm_flax.py)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "kfBkuV1ck4rq" - }, - "outputs": [], - "source": [ - "# linear_decay_lr_schedule_fn = optax.linear_schedule(init_value=learning_rate, end_value=0, transition_steps=num_train_steps)" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "9555" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "steps_per_epoch = len(tokenized_datasets[\"train\"]) // total_batch_size\n", - "steps_per_epoch" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "955" - ] - }, - "execution_count": 36, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "warmup_steps = int(0.1 * num_train_steps)\n", - "warmup_steps" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "8" - ] - }, - "execution_count": 37, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "total_batch_size" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [], - "source": [ - "from typing import Callable\n", - "def create_learning_rate_fn(\n", - " train_ds_size: int, train_batch_size: int, num_train_epochs: int, num_warmup_steps: int, learning_rate: float\n", - ") -> Callable[[int], jnp.array]:\n", - " \"\"\"Returns a linear warmup, linear_decay learning rate function.\"\"\"\n", - " steps_per_epoch = train_ds_size // train_batch_size\n", - " num_train_steps = steps_per_epoch * num_train_epochs\n", - " warmup_fn = optax.linear_schedule(init_value=0.0, end_value=learning_rate, transition_steps=num_warmup_steps)\n", - " decay_fn = optax.linear_schedule(\n", - " init_value=learning_rate, end_value=0, transition_steps=num_train_steps - num_warmup_steps\n", - " )\n", - " schedule_fn = optax.join_schedules(schedules=[warmup_fn, decay_fn], boundaries=[num_warmup_steps])\n", - " return schedule_fn" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [], - "source": [ - "linear_decay_lr_schedule_fn = create_learning_rate_fn(\n", - " len(dataset['train']),\n", - " total_batch_size,\n", - " num_train_epochs,\n", - " warmup_steps,\n", - " learning_rate,\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "2p0yNxeU79F2" - }, - "source": [ - "We will be using the standard Adam optimizer with weight decay, called AdamW (Adam + weight decay). \n", - "\n", - "AdamW can easily be imported from [optax](https://github.com/deepmind/optax) and is created from the just defined learning rate schedule as well as a couple of other hyper-parameters (*beta1*, *beta2*, *epsilon*) that are hard-coded in this notebook.\n", - "\n", - "For more information on AdamW (Adam + weight decay), one can take a look at [this](https://www.fast.ai/2018/07/02/adam-weight-decay/) blog post." - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": { - "id": "xRtpv_iamZd2" - }, - "outputs": [], - "source": [ - "adamw = optax.adamw(learning_rate=linear_decay_lr_schedule_fn, b1=0.9, b2=0.98, eps=1e-8, weight_decay=0.01)" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [], - "source": [ - "adafactor = optax.adafactor(learning_rate=3e-4)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "6g_fEbV-72Hc" - }, - "source": [ - "Next, we will create the *training state* that includes the optimizer, the loss function, and is responsible for updating the model's parameters during training.\n", - "\n", - "Most JAX transformations (notably [jax.jit](https://jax.readthedocs.io/en/latest/jax-101/02-jitting.html)) require functions that are transformed to have no side effects. This is because any such side-effects will only be executed once when the Python version of the function is run during compilation (see [Stateful Computations in JAX](https://jax.readthedocs.io/en/latest/jax-101/07-state.html)). As a consequence, Flax models (which can be transformed by JAX transformations) are **immutable**, and the state of the model (i.e., its weight parameters) is stored *outside* of the model instance.\n", - "\n", - "Models are initialized and updated in a purely functional way: you pass the state to the model when calling it, and the model returns the new (possibly modified) state, leaving the model instance itself unchanged.\n", - "\n", - "Flax provides a convenience class [`flax.training.train_state.TrainState`](https://github.com/google/flax/blob/9da95cdd12591f42d2cd4c17089861bff7e43cc5/flax/training/train_state.py#L22), which stores things such as the model parameters, the loss function, the optimizer, and exposes an `apply_gradients` function to update the model's weight parameters.\n", - "\n", - "Alright, let's begin by defining our *training state* class. We create a `TrainState` class that stores the model's forward pass as the `apply_fn`, the `params`, and the AdamW optimizer." - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": { - "id": "JHYfR67AoKRc" - }, - "outputs": [], - "source": [ - "state = train_state.TrainState.create(apply_fn=model.__call__, params=model.params, tx=adafactor)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "xiYCejDd81TX" - }, - "source": [ - "Next, let's implement a data loader for both training and evaluation.\n", - "The data loader can be defined as a [Python generator](https://wiki.python.org/moin/Generators) that returns a batch model input every time it is called.\n", - "\n", - "First, a random permutation of the whole dataset is defined. \n", - "Then, every time the training data collator is called the next batch of the randomized dataset is extracted, converted to a JAX array and sharded over all local TPU devices." - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": { - "id": "Aos9GltTb3Ve" - }, - "outputs": [], - "source": [ - "def data_loader(rng, dataset, batch_size, shuffle=False):\n", - " steps_per_epoch = len(dataset) // batch_size\n", - "\n", - " if shuffle:\n", - " batch_idx = jax.random.permutation(rng, len(dataset))\n", - " else:\n", - " batch_idx = jnp.arange(len(dataset))\n", - "\n", - " batch_idx = batch_idx[: steps_per_epoch * batch_size] # Skip incomplete batch.\n", - " batch_idx = batch_idx.reshape((steps_per_epoch, batch_size))\n", - "\n", - " for idx in batch_idx:\n", - " batch = dataset[idx]\n", - " batch = {k: jnp.array(v) for k, v in batch.items()}\n", - "\n", - " batch = shard(batch)\n", - "\n", - " yield batch" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "L7uoTXDLUzb-" - }, - "source": [ - "At each training epoch, the dataset should be shuffled and superfluous samples that make the dataset not evenly divisible by the batch size are thrown away. Instead of passing the dataset, we prepare the indices of data samples to be used for both each training epoch. \n", - "The indices for the training dataset are additionally randomly shuffled before each epoch." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "MU6idLb29xYu" - }, - "source": [ - "During fine-tuning, we want to update the model parameters and evaluate the performance after each epoch. \n", - "\n", - "Let's write the functions `train_step` and `eval_step` accordingly. During training the weight parameters should be updated as follows:\n", - "\n", - "1. Define a loss function `loss_function` that first runs a forward pass of the model given data input. Remember that Flax models are immutable, and we explicitly pass it the state (in this case the model parameters and the RNG). `loss_function` returns a scalar loss (using the previously defined `state.loss_function`) between the model output and input targets.\n", - "2. Differentiate this loss function using [`jax.value_and_grad`](https://jax.readthedocs.io/en/latest/notebooks/autodiff_cookbook.html#evaluate-a-function-and-its-gradient-using-value-and-grad). This is a JAX transformation called [automatic differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation), which computes the gradient of `loss_function` given the input to the function (i.e., the parameters of the model), and returns the value and the gradient in a pair `(loss, gradients)`.\n", - "3. Compute the mean gradient over all devices using the collective operation [lax.pmean](https://jax.readthedocs.io/en/latest/_autosummary/jax.lax.pmean.html). As we will see below, each device runs `train_step` on a different batch of data, but by taking the mean here we ensure the model parameters are the same on all devices.\n", - "4. Use `state.apply_gradients`, which applies the gradients to the weights.\n", - "\n", - "Below, you can see how each of the described steps above is put into practice.\n", - "\n", - "Also note that the `labels` are shifted one to the left and the last token of the `logits` is cut. This way, the model learns to predict the **next** token as defined in causal language modeling." - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": { - "id": "GjKzb0zJd-aH" - }, - "outputs": [], - "source": [ - "def train_step(state, batch, dropout_rng):\n", - " dropout_rng, new_dropout_rng = jax.random.split(dropout_rng)\n", - "\n", - " def loss_fn(params):\n", - " labels = batch.pop(\"labels\")\n", - " logits = state.apply_fn(**batch, params=params, dropout_rng=dropout_rng, train=True)[0]\n", - " \n", - " loss = optax.softmax_cross_entropy(logits[..., :-1, :], onehot(labels[..., 1:], logits.shape[-1])).mean()\n", - " return loss\n", - "\n", - " grad_fn = jax.value_and_grad(loss_fn)\n", - " loss, grad = grad_fn(state.params)\n", - " grad = jax.lax.pmean(grad, \"batch\")\n", - " new_state = state.apply_gradients(grads=grad)\n", - "\n", - " metrics = jax.lax.pmean(\n", - " {\"loss\": loss, \"learning_rate\": linear_decay_lr_schedule_fn(state.step)}, axis_name=\"batch\"\n", - " )\n", - "\n", - " return new_state, metrics, new_dropout_rng" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "nCPedI-B-FMQ" - }, - "source": [ - "Now, we want to do parallelized training over all TPU devices. To do so, we use [`jax.pmap`](https://jax.readthedocs.io/en/latest/jax.html?highlight=pmap#parallelization-pmap). This will compile the function once and run the same program on each device (it is an [SPMD program](https://en.wikipedia.org/wiki/SPMD)). When calling this pmapped function, all inputs (`\"state\"`, `\"batch\"`, `\"dropout_rng\"`) should be replicated for all devices, which means that the first axis of each argument is used to map over all TPU devices." - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": { - "id": "w3k1Lqerpw5k" - }, - "outputs": [], - "source": [ - "parallel_train_step = jax.pmap(train_step, \"batch\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "0DWFAZM6A8uf" - }, - "source": [ - "Similarly, we can now define the evaluation step. Here, the function is much easier as we don't need to compute any gradients. To better monitor the performance improvement during training, the next token loss is computed and stored in a `metric` dictionary during evaluation." - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": { - "id": "EGEv7dyfpW4p" - }, - "outputs": [], - "source": [ - "def eval_step(params, batch):\n", - " labels = batch.pop(\"labels\")\n", - "\n", - " logits = model(**batch, params=params, train=False)[0]\n", - "\n", - " loss = optax.softmax_cross_entropy(logits[..., :-1, :], onehot(labels[..., 1:], logits.shape[-1])).mean()\n", - "\n", - " # summarize metrics\n", - " metrics = {\"loss\": loss, \"perplexity\": jnp.exp(loss)}\n", - " metrics = jax.lax.pmean(metrics, axis_name=\"batch\")\n", - " return metrics" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "guaYWTvFA_66" - }, - "source": [ - "Similarly, we also apply `jax.pmap` to the evaluation step." - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": { - "id": "0B8U2r2RpzjV" - }, - "outputs": [], - "source": [ - "parallel_eval_step = jax.pmap(eval_step, \"batch\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "DLaM60PCY8Ka" - }, - "source": [ - "Next, we replicate/copy the weight parameters on each device, so that we can pass them to our parallelized mapped functions." - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "kncZTfALp3PG", - "outputId": "3ce8ee5a-7bda-4ba9-8774-c52a363a98f5" - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/arto/jenv/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.\n", - " warnings.warn(\n", - "/home/arto/jenv/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.\n", - " warnings.warn(\n" - ] - } - ], - "source": [ - "state = flax.jax_utils.replicate(state)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "i2xg8oI-ZJ3P" - }, - "source": [ - "We can almost start training! In a final preparation step, we generate a seeded [**PRNGKey**](https://jax.readthedocs.io/en/latest/_autosummary/jax.random.PRNGKey.html#jax-random-prngkey) used as the random seed for dropout layers and dataset shuffling.\n", - "\n", - "Similar to how we had to copy/replicate the state on all 8 TPU devices, we also need to generate one `PRNGKey` per device, which is why we split the initial `rng` key into 8 random seeds. " - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": { - "id": "idu3E9ubqZH3" - }, - "outputs": [], - "source": [ - "rng = jax.random.PRNGKey(training_seed)\n", - "dropout_rngs = jax.random.split(rng, jax.local_device_count())" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "bKuMWHicbede" - }, - "source": [ - "Now, we are all set to finally start training! \n", - "Let's put all the pieces together and write the training loop. \n", - "\n", - "We start each epoch by generating a new random seed that will be used for dataset shuffling, the dropout layers and the input token masking. \n", - "\n", - "Next, we generate the training dataset indices.\n", - "In the first nested loop - the training loop - we shard the input batch on all 8 TPU devices, and run the training step. \n", - "\n", - "Analogs, in the second nested loop - the evaluation loop - the evaluation batches are sharded and the evaluation step is run.\n", - "\n", - "**Note**: It might seem that the following cell \"hangs\" when executed for the first time. This is because JAX first traces & compiles the code, the very first time it is run. After the first training step, you should notice that execution is much faster." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "eval_loader = data_loader(input_rng, tokenized_datasets[\"validation\"], total_batch_size)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "batch = next(iter(eval_loader))\n", - "batch" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "batch['labels'].shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# labels = batch.pop('labels')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "eval_metric = parallel_eval_step(state.params, model_inputs)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "eval_metric" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "state, train_metric, dropout_rngs = parallel_train_step(state, batch, dropout_rngs)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "train_metric" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 427, - "referenced_widgets": [ - "8c156c360afb4ddc962b87577e093cc4", - "9808f288735b4f5d9eea7377d4603ccc", - "68681edff0db43559012b6653571a289", - "125e8d57908f4679824c96d5247f7119", - "8134ef53fa8044c99b537386e88dbfc9", - "9b35ba10c85048ce8044b87ab9948611", - "e246db4008b14a23b55d2820be3b1ed8", - "b08557f6936e4af9a23f659649979c8d", - "dac789bcf81e4aac86717438ed6e8bbe", - "a1452333389441eab2c8016cd019d2f5", - "799b382dd6c74d528cf77813f82a7335", - "51d0dfb67bbd47ff88964bb446a91967", - "940ce122551e42509e8cec0659e47c77", - "a4234dc426d3459a93263c8b5fe5a34b", - "3549bf51a354408b9a39cb4532957617", - "7f05c774880548d2b73e69fb847c0a2e", - "71fa85dc266149e28685b1f5252185a5", - "9cb943192a154dbc8c7961f128449bbd", - "9de6ea64b67c443fbce1fd6aef4a7409", - "9658cc6977bb4fdf8a081aa99c300872", - "07b26680343f42fc805a3b52b398a5f2", - "97216855bb7f483882afae68bf78a51d", - "a1b9aa80b73748f18e3938df073b3d22", - "ce70b524ffab4052912d47b8adee6748", - "3c5df3c942ca4c768926b431064be351", - "f2e66aebdb4d4812a1f61c1ff990390a", - "269cc543a69e4ba3bae7f0bef67f1e25", - "8cb8398bb3c94284ac2ff42c7b3bc7c5", - "0b898451f79c4b6eb1678a8f366a5cbf", - "7a5426d6898e4ba2b5e3c48a65096232", - "2cf7d4c7dc6f413eb8cc9460e5de8d93", - "f7851bd9702949ee8e6278ad69c9d981", - "ab08bd64073d4d488f9c41e8595bac08", - "5e1805a3c7f24c3f9f15e86d15462d44", - "fc2e1a72620742499457ab2e31da12a9", - "b3b4d96626044375bad11ba48cc25ee3", - "c55e4e5f6d0a42b4ac25fb6d68a8e842", - "c7cca560cd4340bb986297655f513746", - "cd4892863d524637b8e567e2beed23bb", - "c85760cb6ccb4020be999357f2501bf9", - "7d0775fcf6a8497db20326dc94bd2739", - "5215d791d9144c089b6996336c3085cd", - "839daa7d5cdc4669942f1b5ef5bf4903", - "4bc03fce15a14355b3429c138ea1eadd", - "d7cddd2cafe84d17829096b31ee5e93e", - "b4ce8892d5784011a649eeca7b17fd0d", - "7a7b598979174b338f8fdfc90af61e47", - "cd946e84023445d59eea9893fba02f1a", - "4ec581224b3f47f2acdfaeb740c94526", - "81c25bb635db4bc9a62636bf3a3868eb", - "1462263d23c340e781d596a3f5414e03", - "c8bd20a456cc4783a2fbbdec9a2880ea", - "f0aa35a704a84967a3471e5f698bb04c", - "f2c541b4ecd04729a518414f8b878099", - "402795b6506a45c9831fb0ddb2ce0749", - "0ba00c3340414029938c9c947d4a74b6", - "d17f610741f142b782c74ac4e04e6481", - "84c8a7cbfdb04690941f8c32299744bb", - "f3f0b212907843f5be2131b7b60a955d", - "0a6a241c99bb49ad9baaef052b157964", - "b642c546813240a499a165279c743e7a", - "6acb415dc8b947bb85ab8f8959bc10e1", - "2c311c4506264995b7fbaba02333e747", - "2fde17ab63e5444799ec1dc5501d946e", - "6173f15a3e5a40819bd4f3833544fd78", - "da2f69c01bcd4139980a84ddca297817", - "b9470519dfff45228e54d0181337ee17", - "b5824ca248b74052bb1696b47369838e", - "11f8b1ddd5af4bb3babd492bbaf3fb7d", - "861e2a6f8e3e4a5cacfd0a746180cc90", - "181f0680e5bb4d23a69f9cc747207883", - "0e2531c122b44823a012ba844066e60c", - "a2324cab4e9745ddad69b35e77ed148e", - "42c8cb98eaae4cee86d49c675d3a3604", - "146ce87ac5b445e385285bdbbd70fba9", - "46340cbdaf9c47719e005b42b7236be2", - "cd26f91d46ef42609d265f0b709c3401", - "6f5f003a277244b2824e09b877fefa74", - "57a93be7bb3d4167a89f7d73d5915551", - "db96a8d67e564cd58358916ddfb86948", - "194f5ce951d54193b887f1cc91e3640b", - "d8770c573383466f81d7dfd01f048677", - "1fe6ad92e6834142844bb32e7a0d424d", - "9d905b40bf2746e79abad7257cf8654f", - "bfa957f8482847978542373db16cfa8a", - "1f836daeb9b7495a9ae00bc2c28fa212", - "0973226c646f4bb28f291ed3eb4e5cba", - "fd72a18adfb34bc2b2dab769067dcc1a", - "91fb9675236949e6bb8c99015061647c", - "75f988a47d774cd38fd85977fda7a96c", - "503ff750d98d4f2c8f7af32adc221685", - "8b3645c111b842f4b1d9ff54bb94f26e", - "dabd19b76eec4139a3028aa1776ac496", - "b6410389dafe43bd885446a6d2d255b0", - "23187b20efd44965966d6576c0d057aa", - "6f876daff243410bb26b8feb1cf981e0", - "2374ba361eaa41899b7eabe7da829259", - "522331eac9cb49e4998384f42eeba912", - "6eaf7a1f2e5e4aa590da55adf969ecc6", - "29b0ec79b8134168a5e3ae382088c147", - "96f9368df9dc453495badb09e767d091", - "02b1a546bc334efa93ba8dff5890dd0c", - "b7f8df98ab5d45f39b34ceb1018c01b5", - "c251cf1c33c741049cc9710e89f09bc0", - "7e067f02bdb74846a42909cd9ad1961b", - "68cf981867f7461c999c9e9f85af6499", - "b6eec0df5ca64d5dbf32cb5231127e3b", - "6efef5f4637e4efaa373cb9214d4e911", - "d539e4c93f4941e9af0b742df4c53ae0", - "dafd837bebd44c00ae68641cc0b31d3d", - "47808ac98d064711bccc77037e53dade", - "acfab7adb0ed440b87fa08f81086522d", - "6fa186bd796b4d16bfb67657178e4f65", - "07fe952dec2b4f2687bcddec8701a70e", - "6c9e7145c2d7439db6f63b29c505e63e", - "43861346cadc460992ab7e2c30aa556c", - "6baec27867e943a2bad6e70e48978ea5", - "d2e74c9d62444e98b52078d6a15dd068", - "fcf96c9bb04642089466de2421bd1af2", - "b9d1f631577f4a2a8eb49d3f8535ba25", - "e9c8d09e028741979a764789fd7f256e", - "2bd15c15e3b748cd9ee001bb86c04b6d", - "f5e27446f61747e1a65e477e2975dbcf", - "1e17af39c928463b800d22ad1fe533e5", - "98e148811880459ea7a7641ee3a301b6", - "6669a1844b394ea499dc92e126132bfc", - "987310405403439893813fb8c5da94a1", - "f48e444c929748b2a875c6eecc028bc4", - "1ed01b7ba2a64bebbbf0e641c8d1560e", - "bfadf2c9e99c48e4860c2a67b2955120", - "63f47b7d6bc849d1b869367a6ba8b40f", - "1bfe0a0ab5694e1d857ea981090d4f09", - "7f6628d5f61f424ba6373da2f6defe76", - "01d1c1e078e7403daabd1e7d8d5a4fda", - "3b802a49c0824b968d88350f82ff8422", - "5a6ea875aeff4688ba37481af5b5d9fe", - "6dfceed81b9b4f8198a8186dfdc72d42", - "44ee8edfb7c744358f0cb4567535ce07", - "5eb4c37fcf6a4b618a8f573b837a8c3a", - "5bc32858674f442fa7b49efcccfc3c15", - "3ebb7960558247ec84ef155042d2077a", - "52a25fde354e45dd9635097d22c16def", - "ce3117149c7e4ae6a93fbe959bd3f543", - "900aa99ccfeb4a38a0ad90f8ce656fe6", - "b414ff8df761462e8df8f51788f69508", - "e0c2ef0fd4d544b9b80b00689d21f503", - "52cae29e50974a42a3882f91b2a8f2ba", - "02b454a4d7ce44b0b819fc7085b1d357", - "6321a346b671411791979a1a343fb494", - "22941d4a640b4d8bba1b97f723da0ef0", - "b6f9da40231f4404bb0c90458c94c5c2", - "7db664fc575747af8e10ad6f2783a166", - "62fd4952400246189945f2de7f0192c6", - "f07d424e031d457a98a88b94b0c844b8", - "a9f8af0a5e594c29b553983b6a1c4c7d", - "5829f975dedd45d3b315ab29c1b12253", - "2eb6756f48074993825b5bfac47641e1", - "389fcde31cf24c438a9fcdb155060495", - "a5a86539a58f4fe98ba96f9cd3d46588", - "32c067d64cb7445eb0464f3a10071e7c", - "c014bf31dccb4448b561a702361ed9bc", - "ef823903b36348978833920bb7e03881", - "c6855514c5a649ebadc2db53c267e9e0", - "8602122a90db462cacef9373fca054fe", - "892e02ac20ef4963871d915085139f7d", - "173704b44e2c472499bc4242ef9e0863", - "2f647e8c1be2464fb9c69e8af5e5bf5a", - "5fc3a90707f746c884d924e385b0cc1c" - ] - }, - "id": "U946A-YZp-Pe", - "outputId": "1fe21ffd-2e39-4470-fb87-6b11ab1d4024" - }, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "17551e2885a240c3be0f5ee31e96aead", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Epoch ...: 0%| | 0/1 [00:00\u001b[0;34m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mmodel_inputs\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtrain_loader\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;31m# Model forward\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_metric\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdropout_rngs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparallel_train_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdropout_rngs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0mprogress_bar_train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - " \u001b[0;31m[... skipping hidden 1 frame]\u001b[0m\n", - "\u001b[0;32m~/jenv/lib/python3.8/site-packages/jax/_src/api.py\u001b[0m in \u001b[0;36mf_pmapped\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 1645\u001b[0m list(out_axes_leaves)))),\n\u001b[1;32m 1646\u001b[0m closure=(tuple(out_axes_leaves), out_axes_treedef))\n\u001b[0;32m-> 1647\u001b[0;31m out = pxla.xla_pmap(\n\u001b[0m\u001b[1;32m 1648\u001b[0m \u001b[0mflat_fun\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbackend\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbackend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1649\u001b[0m \u001b[0maxis_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlocal_axis_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mglobal_axis_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/jenv/lib/python3.8/site-packages/jax/core.py\u001b[0m in \u001b[0;36mbind\u001b[0;34m(self, fun, *args, **params)\u001b[0m\n\u001b[1;32m 1618\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mbind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfun\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1619\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'in_axes'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1620\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mcall_bind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfun\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1621\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1622\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfun\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtracers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/jenv/lib/python3.8/site-packages/jax/core.py\u001b[0m in \u001b[0;36mcall_bind\u001b[0;34m(primitive, fun, *args, **params)\u001b[0m\n\u001b[1;32m 1549\u001b[0m params_tuple, out_axes_transforms)\n\u001b[1;32m 1550\u001b[0m \u001b[0mtracers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtop_trace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfull_raise\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1551\u001b[0;31m \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mprimitive\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtop_trace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfun\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtracers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1552\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfull_lower\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mapply_todos\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0menv_trace_todo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1553\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/jenv/lib/python3.8/site-packages/jax/core.py\u001b[0m in \u001b[0;36mprocess\u001b[0;34m(self, trace, fun, tracers, params)\u001b[0m\n\u001b[1;32m 1621\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1622\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfun\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtracers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1623\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mtrace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprocess_map\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfun\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtracers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1624\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1625\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mpost_process\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout_tracers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/jenv/lib/python3.8/site-packages/jax/core.py\u001b[0m in \u001b[0;36mprocess_call\u001b[0;34m(self, primitive, f, tracers, params)\u001b[0m\n\u001b[1;32m 604\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 605\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mprocess_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprimitive\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtracers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 606\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mprimitive\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimpl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mtracers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 607\u001b[0m \u001b[0mprocess_map\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mprocess_call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 608\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/jenv/lib/python3.8/site-packages/jax/interpreters/pxla.py\u001b[0m in \u001b[0;36mxla_pmap_impl\u001b[0;34m(fun, backend, axis_name, axis_size, global_axis_size, devices, name, in_axes, out_axes_thunk, donated_invars, global_arg_shapes, *args)\u001b[0m\n\u001b[1;32m 635\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m\"abstract args\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxla\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mabstractify\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 636\u001b[0m (\"fingerprint\", fingerprint))\n\u001b[0;32m--> 637\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mcompiled_fun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 638\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 639\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mlu\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcache\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/jenv/lib/python3.8/site-packages/jax/interpreters/pxla.py\u001b[0m in \u001b[0;36mexecute_replicated\u001b[0;34m(compiled, backend, in_handler, out_handler, *args)\u001b[0m\n\u001b[1;32m 1150\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mexecute_replicated\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcompiled\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbackend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0min_handler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout_handler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1151\u001b[0m \u001b[0minput_bufs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0min_handler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1152\u001b[0;31m \u001b[0mout_bufs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompiled\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute_sharded_on_local_devices\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_bufs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1153\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mxla\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mneeds_check_special\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1154\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mbufs\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mout_bufs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] - } - ], - "source": [ - "for epoch in tqdm(range(1, num_train_epochs + 1), desc=f\"Epoch ...\", position=0, leave=True):\n", - " rng, input_rng = jax.random.split(rng)\n", - "\n", - " # -- Train --\n", - " train_loader = data_loader(input_rng, tokenized_datasets[\"train\"], total_batch_size, shuffle=True)\n", - " with tqdm(total=len(tokenized_datasets[\"train\"]) // total_batch_size, desc=\"Training...\", leave=False) as progress_bar_train:\n", - " for model_inputs in train_loader:\n", - " # Model forward\n", - " state, train_metric, dropout_rngs = parallel_train_step(state, model_inputs, dropout_rngs)\n", - "\n", - " progress_bar_train.update(1)\n", - "\n", - " progress_bar_train.write(\n", - " f\"Train... ({epoch}/{num_epochs} | Loss: {round(train_metric['loss'].mean(), 3)}, Learning Rate: {round(train_metric['learning_rate'].mean(), 6)})\"\n", - " )\n", - "\n", - " # -- Eval --\n", - " eval_loader = data_loader(input_rng, tokenized_datasets[\"validation\"], total_batch_size)\n", - " eval_metrics = []\n", - " \n", - " with tqdm(total=len(tokenized_datasets[\"validation\"]) // total_batch_size, desc=\"Evaluation...\", leave=False) as progress_bar_eval:\n", - " for model_inputs in eval_loader:\n", - " # Model forward\n", - " eval_metric = parallel_eval_step(state.params, model_inputs)\n", - " eval_metrics.append(eval_metric)\n", - "\n", - " progress_bar_eval.update(1)\n", - " \n", - " eval_metrics = get_metrics(eval_metrics)\n", - " eval_metrics = jax.tree_map(jnp.mean, eval_metrics)\n", - " progress_bar_eval.write(\n", - " f\"Eval... ({epoch}/{num_epochs} | Loss: {eval_metrics['loss']} | Perplexity: {eval_metrics['perplexity']})\"\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ZI4XIhY-7hyh" - }, - "source": [ - "It can be seen that in this colab training already reaches a speed of 2.42 training steps per second. Executing [**`run_clm_flax.py`**](https://github.com/huggingface/transformers/tree/master/examples/flax/language-modeling/run_clm_flax.py) on a TPUv3-8 VM should be as fast as 7 training steps per second.\n", - "\n", - "For a more in-detail comparison of runtimes please refer to [this](https://github.com/huggingface/transformers/tree/master/examples/flax/language-modeling#runtime-evaluation) table." - ] - } - ], - "metadata": { - "accelerator": "TPU", - "colab": { - "collapsed_sections": [], - "name": "Causal Language Model Training on TPU with 🤗 Transformers & JAX", - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "01d1c1e078e7403daabd1e7d8d5a4fda": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "02b1a546bc334efa93ba8dff5890dd0c": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "02b454a4d7ce44b0b819fc7085b1d357": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_7db664fc575747af8e10ad6f2783a166", - "placeholder": "​", - "style": "IPY_MODEL_b6f9da40231f4404bb0c90458c94c5c2", - "value": " 12/12 [00:05<00:00, 2.35it/s]" - } - }, - "07b26680343f42fc805a3b52b398a5f2": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "07fe952dec2b4f2687bcddec8701a70e": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "0973226c646f4bb28f291ed3eb4e5cba": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "0a6a241c99bb49ad9baaef052b157964": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_2fde17ab63e5444799ec1dc5501d946e", - "placeholder": "​", - "style": "IPY_MODEL_2c311c4506264995b7fbaba02333e747", - "value": " 137/137 [01:38<00:00, 1.40it/s]" - } - }, - "0b898451f79c4b6eb1678a8f366a5cbf": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "0ba00c3340414029938c9c947d4a74b6": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "0e2531c122b44823a012ba844066e60c": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "11f8b1ddd5af4bb3babd492bbaf3fb7d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "125e8d57908f4679824c96d5247f7119": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_b08557f6936e4af9a23f659649979c8d", - "placeholder": "​", - "style": "IPY_MODEL_e246db4008b14a23b55d2820be3b1ed8", - "value": " 10/10 [19:05<00:00, 114.56s/it]" - } - }, - "1462263d23c340e781d596a3f5414e03": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "", - "description": "Evaluation...: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_f2c541b4ecd04729a518414f8b878099", - "max": 12, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_f0aa35a704a84967a3471e5f698bb04c", - "value": 12 - } - }, - "146ce87ac5b445e385285bdbbd70fba9": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "", - "description": "Training...: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_6f5f003a277244b2824e09b877fefa74", - "max": 137, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_cd26f91d46ef42609d265f0b709c3401", - "value": 137 - } - }, - "173704b44e2c472499bc4242ef9e0863": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "181f0680e5bb4d23a69f9cc747207883": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "194f5ce951d54193b887f1cc91e3640b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_1fe6ad92e6834142844bb32e7a0d424d", - "IPY_MODEL_9d905b40bf2746e79abad7257cf8654f" - ], - "layout": "IPY_MODEL_d8770c573383466f81d7dfd01f048677" - } - }, - "1bfe0a0ab5694e1d857ea981090d4f09": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_5a6ea875aeff4688ba37481af5b5d9fe", - "placeholder": "​", - "style": "IPY_MODEL_3b802a49c0824b968d88350f82ff8422", - "value": " 12/12 [00:05<00:00, 2.35it/s]" - } - }, - "1e17af39c928463b800d22ad1fe533e5": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_f48e444c929748b2a875c6eecc028bc4", - "placeholder": "​", - "style": "IPY_MODEL_987310405403439893813fb8c5da94a1", - "value": " 137/137 [01:36<00:00, 1.41it/s]" - } - }, - "1ed01b7ba2a64bebbbf0e641c8d1560e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_63f47b7d6bc849d1b869367a6ba8b40f", - "IPY_MODEL_1bfe0a0ab5694e1d857ea981090d4f09" - ], - "layout": "IPY_MODEL_bfadf2c9e99c48e4860c2a67b2955120" - } - }, - "1f836daeb9b7495a9ae00bc2c28fa212": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "1fe6ad92e6834142844bb32e7a0d424d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "", - "description": "Evaluation...: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_1f836daeb9b7495a9ae00bc2c28fa212", - "max": 12, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_bfa957f8482847978542373db16cfa8a", - "value": 12 - } - }, - "22941d4a640b4d8bba1b97f723da0ef0": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "23187b20efd44965966d6576c0d057aa": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "2374ba361eaa41899b7eabe7da829259": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_6eaf7a1f2e5e4aa590da55adf969ecc6", - "IPY_MODEL_29b0ec79b8134168a5e3ae382088c147" - ], - "layout": "IPY_MODEL_522331eac9cb49e4998384f42eeba912" - } - }, - "269cc543a69e4ba3bae7f0bef67f1e25": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "", - "description": "Training...: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_7a5426d6898e4ba2b5e3c48a65096232", - "max": 137, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_0b898451f79c4b6eb1678a8f366a5cbf", - "value": 137 - } - }, - "29b0ec79b8134168a5e3ae382088c147": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_c251cf1c33c741049cc9710e89f09bc0", - "placeholder": "​", - "style": "IPY_MODEL_b7f8df98ab5d45f39b34ceb1018c01b5", - "value": " 12/12 [00:05<00:00, 2.35it/s]" - } - }, - "2bd15c15e3b748cd9ee001bb86c04b6d": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "2c311c4506264995b7fbaba02333e747": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "2cf7d4c7dc6f413eb8cc9460e5de8d93": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "2eb6756f48074993825b5bfac47641e1": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "2f647e8c1be2464fb9c69e8af5e5bf5a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "2fde17ab63e5444799ec1dc5501d946e": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "32c067d64cb7445eb0464f3a10071e7c": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "3549bf51a354408b9a39cb4532957617": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "389fcde31cf24c438a9fcdb155060495": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "3b802a49c0824b968d88350f82ff8422": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "3c5df3c942ca4c768926b431064be351": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_269cc543a69e4ba3bae7f0bef67f1e25", - "IPY_MODEL_8cb8398bb3c94284ac2ff42c7b3bc7c5" - ], - "layout": "IPY_MODEL_f2e66aebdb4d4812a1f61c1ff990390a" - } - }, - "3ebb7960558247ec84ef155042d2077a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "402795b6506a45c9831fb0ddb2ce0749": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "42c8cb98eaae4cee86d49c675d3a3604": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "43861346cadc460992ab7e2c30aa556c": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_b9d1f631577f4a2a8eb49d3f8535ba25", - "placeholder": "​", - "style": "IPY_MODEL_fcf96c9bb04642089466de2421bd1af2", - "value": " 12/12 [00:05<00:00, 2.33it/s]" - } - }, - "44ee8edfb7c744358f0cb4567535ce07": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "46340cbdaf9c47719e005b42b7236be2": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_db96a8d67e564cd58358916ddfb86948", - "placeholder": "​", - "style": "IPY_MODEL_57a93be7bb3d4167a89f7d73d5915551", - "value": " 137/137 [01:37<00:00, 1.40it/s]" - } - }, - "47808ac98d064711bccc77037e53dade": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "4bc03fce15a14355b3429c138ea1eadd": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_cd946e84023445d59eea9893fba02f1a", - "placeholder": "​", - "style": "IPY_MODEL_7a7b598979174b338f8fdfc90af61e47", - "value": " 137/137 [01:37<00:00, 1.41it/s]" - } - }, - "4ec581224b3f47f2acdfaeb740c94526": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_1462263d23c340e781d596a3f5414e03", - "IPY_MODEL_c8bd20a456cc4783a2fbbdec9a2880ea" - ], - "layout": "IPY_MODEL_81c25bb635db4bc9a62636bf3a3868eb" - } - }, - "503ff750d98d4f2c8f7af32adc221685": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "", - "description": "Training...: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_b6410389dafe43bd885446a6d2d255b0", - "max": 137, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_dabd19b76eec4139a3028aa1776ac496", - "value": 137 - } - }, - "51d0dfb67bbd47ff88964bb446a91967": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_7f05c774880548d2b73e69fb847c0a2e", - "placeholder": "​", - "style": "IPY_MODEL_3549bf51a354408b9a39cb4532957617", - "value": " 137/137 [03:28<00:00, 1.41it/s]" - } - }, - "5215d791d9144c089b6996336c3085cd": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "522331eac9cb49e4998384f42eeba912": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "52a25fde354e45dd9635097d22c16def": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "52cae29e50974a42a3882f91b2a8f2ba": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "", - "description": "Evaluation...: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_22941d4a640b4d8bba1b97f723da0ef0", - "max": 12, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_6321a346b671411791979a1a343fb494", - "value": 12 - } - }, - "57a93be7bb3d4167a89f7d73d5915551": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "5829f975dedd45d3b315ab29c1b12253": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_32c067d64cb7445eb0464f3a10071e7c", - "placeholder": "​", - "style": "IPY_MODEL_a5a86539a58f4fe98ba96f9cd3d46588", - "value": " 137/137 [01:37<00:00, 1.40it/s]" - } - }, - "5a6ea875aeff4688ba37481af5b5d9fe": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "5bc32858674f442fa7b49efcccfc3c15": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_900aa99ccfeb4a38a0ad90f8ce656fe6", - "placeholder": "​", - "style": "IPY_MODEL_ce3117149c7e4ae6a93fbe959bd3f543", - "value": " 137/137 [01:37<00:00, 1.42it/s]" - } - }, - "5e1805a3c7f24c3f9f15e86d15462d44": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "5eb4c37fcf6a4b618a8f573b837a8c3a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "", - "description": "Training...: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_52a25fde354e45dd9635097d22c16def", - "max": 137, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_3ebb7960558247ec84ef155042d2077a", - "value": 137 - } - }, - "5fc3a90707f746c884d924e385b0cc1c": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "6173f15a3e5a40819bd4f3833544fd78": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_b9470519dfff45228e54d0181337ee17", - "IPY_MODEL_b5824ca248b74052bb1696b47369838e" - ], - "layout": "IPY_MODEL_da2f69c01bcd4139980a84ddca297817" - } - }, - "62fd4952400246189945f2de7f0192c6": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_a9f8af0a5e594c29b553983b6a1c4c7d", - "IPY_MODEL_5829f975dedd45d3b315ab29c1b12253" - ], - "layout": "IPY_MODEL_f07d424e031d457a98a88b94b0c844b8" - } - }, - "6321a346b671411791979a1a343fb494": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "63f47b7d6bc849d1b869367a6ba8b40f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "", - "description": "Evaluation...: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_01d1c1e078e7403daabd1e7d8d5a4fda", - "max": 12, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_7f6628d5f61f424ba6373da2f6defe76", - "value": 12 - } - }, - "6669a1844b394ea499dc92e126132bfc": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "68681edff0db43559012b6653571a289": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "Epoch ...: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_9b35ba10c85048ce8044b87ab9948611", - "max": 10, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_8134ef53fa8044c99b537386e88dbfc9", - "value": 10 - } - }, - "68cf981867f7461c999c9e9f85af6499": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "6acb415dc8b947bb85ab8f8959bc10e1": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "6baec27867e943a2bad6e70e48978ea5": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "6c9e7145c2d7439db6f63b29c505e63e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "", - "description": "Evaluation...: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_d2e74c9d62444e98b52078d6a15dd068", - "max": 12, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_6baec27867e943a2bad6e70e48978ea5", - "value": 12 - } - }, - "6dfceed81b9b4f8198a8186dfdc72d42": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_5eb4c37fcf6a4b618a8f573b837a8c3a", - "IPY_MODEL_5bc32858674f442fa7b49efcccfc3c15" - ], - "layout": "IPY_MODEL_44ee8edfb7c744358f0cb4567535ce07" - } - }, - "6eaf7a1f2e5e4aa590da55adf969ecc6": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "", - "description": "Evaluation...: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_02b1a546bc334efa93ba8dff5890dd0c", - "max": 12, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_96f9368df9dc453495badb09e767d091", - "value": 12 - } - }, - "6efef5f4637e4efaa373cb9214d4e911": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_acfab7adb0ed440b87fa08f81086522d", - "placeholder": "​", - "style": "IPY_MODEL_47808ac98d064711bccc77037e53dade", - "value": " 137/137 [01:36<00:00, 1.42it/s]" - } - }, - "6f5f003a277244b2824e09b877fefa74": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "6f876daff243410bb26b8feb1cf981e0": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "6fa186bd796b4d16bfb67657178e4f65": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_6c9e7145c2d7439db6f63b29c505e63e", - "IPY_MODEL_43861346cadc460992ab7e2c30aa556c" - ], - "layout": "IPY_MODEL_07fe952dec2b4f2687bcddec8701a70e" - } - }, - "71fa85dc266149e28685b1f5252185a5": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_9de6ea64b67c443fbce1fd6aef4a7409", - "IPY_MODEL_9658cc6977bb4fdf8a081aa99c300872" - ], - "layout": "IPY_MODEL_9cb943192a154dbc8c7961f128449bbd" - } - }, - "75f988a47d774cd38fd85977fda7a96c": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "799b382dd6c74d528cf77813f82a7335": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "", - "description": "Training...: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_a4234dc426d3459a93263c8b5fe5a34b", - "max": 137, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_940ce122551e42509e8cec0659e47c77", - "value": 137 - } - }, - "7a5426d6898e4ba2b5e3c48a65096232": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "7a7b598979174b338f8fdfc90af61e47": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "7d0775fcf6a8497db20326dc94bd2739": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_839daa7d5cdc4669942f1b5ef5bf4903", - "IPY_MODEL_4bc03fce15a14355b3429c138ea1eadd" - ], - "layout": "IPY_MODEL_5215d791d9144c089b6996336c3085cd" - } - }, - "7db664fc575747af8e10ad6f2783a166": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "7e067f02bdb74846a42909cd9ad1961b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_b6eec0df5ca64d5dbf32cb5231127e3b", - "IPY_MODEL_6efef5f4637e4efaa373cb9214d4e911" - ], - "layout": "IPY_MODEL_68cf981867f7461c999c9e9f85af6499" - } - }, - "7f05c774880548d2b73e69fb847c0a2e": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "7f6628d5f61f424ba6373da2f6defe76": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "8134ef53fa8044c99b537386e88dbfc9": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "81c25bb635db4bc9a62636bf3a3868eb": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "839daa7d5cdc4669942f1b5ef5bf4903": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "", - "description": "Training...: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_b4ce8892d5784011a649eeca7b17fd0d", - "max": 137, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_d7cddd2cafe84d17829096b31ee5e93e", - "value": 137 - } - }, - "84c8a7cbfdb04690941f8c32299744bb": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "8602122a90db462cacef9373fca054fe": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_5fc3a90707f746c884d924e385b0cc1c", - "placeholder": "​", - "style": "IPY_MODEL_2f647e8c1be2464fb9c69e8af5e5bf5a", - "value": " 12/12 [00:05<00:00, 2.35it/s]" - } - }, - "861e2a6f8e3e4a5cacfd0a746180cc90": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "892e02ac20ef4963871d915085139f7d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "8b3645c111b842f4b1d9ff54bb94f26e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_6f876daff243410bb26b8feb1cf981e0", - "placeholder": "​", - "style": "IPY_MODEL_23187b20efd44965966d6576c0d057aa", - "value": " 137/137 [01:38<00:00, 1.43it/s]" - } - }, - "8c156c360afb4ddc962b87577e093cc4": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_68681edff0db43559012b6653571a289", - "IPY_MODEL_125e8d57908f4679824c96d5247f7119" - ], - "layout": "IPY_MODEL_9808f288735b4f5d9eea7377d4603ccc" - } - }, - "8cb8398bb3c94284ac2ff42c7b3bc7c5": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_f7851bd9702949ee8e6278ad69c9d981", - "placeholder": "​", - "style": "IPY_MODEL_2cf7d4c7dc6f413eb8cc9460e5de8d93", - "value": " 137/137 [01:37<00:00, 1.41it/s]" - } - }, - "900aa99ccfeb4a38a0ad90f8ce656fe6": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "91fb9675236949e6bb8c99015061647c": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_503ff750d98d4f2c8f7af32adc221685", - "IPY_MODEL_8b3645c111b842f4b1d9ff54bb94f26e" - ], - "layout": "IPY_MODEL_75f988a47d774cd38fd85977fda7a96c" - } - }, - "940ce122551e42509e8cec0659e47c77": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "9658cc6977bb4fdf8a081aa99c300872": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_ce70b524ffab4052912d47b8adee6748", - "placeholder": "​", - "style": "IPY_MODEL_a1b9aa80b73748f18e3938df073b3d22", - "value": " 12/12 [00:12<00:00, 1.78it/s]" - } - }, - "96f9368df9dc453495badb09e767d091": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "97216855bb7f483882afae68bf78a51d": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "9808f288735b4f5d9eea7377d4603ccc": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "987310405403439893813fb8c5da94a1": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "98e148811880459ea7a7641ee3a301b6": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "9b35ba10c85048ce8044b87ab9948611": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "9cb943192a154dbc8c7961f128449bbd": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "9d905b40bf2746e79abad7257cf8654f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_fd72a18adfb34bc2b2dab769067dcc1a", - "placeholder": "​", - "style": "IPY_MODEL_0973226c646f4bb28f291ed3eb4e5cba", - "value": " 12/12 [00:05<00:00, 2.35it/s]" - } - }, - "9de6ea64b67c443fbce1fd6aef4a7409": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "", - "description": "Evaluation...: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_97216855bb7f483882afae68bf78a51d", - "max": 12, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_07b26680343f42fc805a3b52b398a5f2", - "value": 12 - } - }, - "a1452333389441eab2c8016cd019d2f5": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a1b9aa80b73748f18e3938df073b3d22": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "a2324cab4e9745ddad69b35e77ed148e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_146ce87ac5b445e385285bdbbd70fba9", - "IPY_MODEL_46340cbdaf9c47719e005b42b7236be2" - ], - "layout": "IPY_MODEL_42c8cb98eaae4cee86d49c675d3a3604" - } - }, - "a4234dc426d3459a93263c8b5fe5a34b": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a5a86539a58f4fe98ba96f9cd3d46588": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "a9f8af0a5e594c29b553983b6a1c4c7d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "", - "description": "Training...: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_389fcde31cf24c438a9fcdb155060495", - "max": 137, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_2eb6756f48074993825b5bfac47641e1", - "value": 137 - } - }, - "ab08bd64073d4d488f9c41e8595bac08": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_fc2e1a72620742499457ab2e31da12a9", - "IPY_MODEL_b3b4d96626044375bad11ba48cc25ee3" - ], - "layout": "IPY_MODEL_5e1805a3c7f24c3f9f15e86d15462d44" - } - }, - "acfab7adb0ed440b87fa08f81086522d": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "b08557f6936e4af9a23f659649979c8d": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "b3b4d96626044375bad11ba48cc25ee3": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_c85760cb6ccb4020be999357f2501bf9", - "placeholder": "​", - "style": "IPY_MODEL_cd4892863d524637b8e567e2beed23bb", - "value": " 12/12 [00:05<00:00, 2.31it/s]" - } - }, - "b414ff8df761462e8df8f51788f69508": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_52cae29e50974a42a3882f91b2a8f2ba", - "IPY_MODEL_02b454a4d7ce44b0b819fc7085b1d357" - ], - "layout": "IPY_MODEL_e0c2ef0fd4d544b9b80b00689d21f503" - } - }, - "b4ce8892d5784011a649eeca7b17fd0d": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "b5824ca248b74052bb1696b47369838e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_0e2531c122b44823a012ba844066e60c", - "placeholder": "​", - "style": "IPY_MODEL_181f0680e5bb4d23a69f9cc747207883", - "value": " 12/12 [00:05<00:00, 2.34it/s]" - } - }, - "b6410389dafe43bd885446a6d2d255b0": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "b642c546813240a499a165279c743e7a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "b6eec0df5ca64d5dbf32cb5231127e3b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "", - "description": "Training...: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_dafd837bebd44c00ae68641cc0b31d3d", - "max": 137, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_d539e4c93f4941e9af0b742df4c53ae0", - "value": 137 - } - }, - "b6f9da40231f4404bb0c90458c94c5c2": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "b7f8df98ab5d45f39b34ceb1018c01b5": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "b9470519dfff45228e54d0181337ee17": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "", - "description": "Evaluation...: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_861e2a6f8e3e4a5cacfd0a746180cc90", - "max": 12, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_11f8b1ddd5af4bb3babd492bbaf3fb7d", - "value": 12 - } - }, - "b9d1f631577f4a2a8eb49d3f8535ba25": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "bfa957f8482847978542373db16cfa8a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "bfadf2c9e99c48e4860c2a67b2955120": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "c014bf31dccb4448b561a702361ed9bc": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_c6855514c5a649ebadc2db53c267e9e0", - "IPY_MODEL_8602122a90db462cacef9373fca054fe" - ], - "layout": "IPY_MODEL_ef823903b36348978833920bb7e03881" - } - }, - "c251cf1c33c741049cc9710e89f09bc0": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "c55e4e5f6d0a42b4ac25fb6d68a8e842": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "c6855514c5a649ebadc2db53c267e9e0": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "", - "description": "Evaluation...: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_173704b44e2c472499bc4242ef9e0863", - "max": 12, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_892e02ac20ef4963871d915085139f7d", - "value": 12 - } - }, - "c7cca560cd4340bb986297655f513746": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "c85760cb6ccb4020be999357f2501bf9": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "c8bd20a456cc4783a2fbbdec9a2880ea": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_0ba00c3340414029938c9c947d4a74b6", - "placeholder": "​", - "style": "IPY_MODEL_402795b6506a45c9831fb0ddb2ce0749", - "value": " 12/12 [00:05<00:00, 2.35it/s]" - } - }, - "cd26f91d46ef42609d265f0b709c3401": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "cd4892863d524637b8e567e2beed23bb": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "cd946e84023445d59eea9893fba02f1a": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "ce3117149c7e4ae6a93fbe959bd3f543": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "ce70b524ffab4052912d47b8adee6748": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "d17f610741f142b782c74ac4e04e6481": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_f3f0b212907843f5be2131b7b60a955d", - "IPY_MODEL_0a6a241c99bb49ad9baaef052b157964" - ], - "layout": "IPY_MODEL_84c8a7cbfdb04690941f8c32299744bb" - } - }, - "d2e74c9d62444e98b52078d6a15dd068": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "d539e4c93f4941e9af0b742df4c53ae0": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "d7cddd2cafe84d17829096b31ee5e93e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "d8770c573383466f81d7dfd01f048677": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "da2f69c01bcd4139980a84ddca297817": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "dabd19b76eec4139a3028aa1776ac496": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "dac789bcf81e4aac86717438ed6e8bbe": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_799b382dd6c74d528cf77813f82a7335", - "IPY_MODEL_51d0dfb67bbd47ff88964bb446a91967" - ], - "layout": "IPY_MODEL_a1452333389441eab2c8016cd019d2f5" - } - }, - "dafd837bebd44c00ae68641cc0b31d3d": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "db96a8d67e564cd58358916ddfb86948": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e0c2ef0fd4d544b9b80b00689d21f503": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e246db4008b14a23b55d2820be3b1ed8": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "e9c8d09e028741979a764789fd7f256e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_f5e27446f61747e1a65e477e2975dbcf", - "IPY_MODEL_1e17af39c928463b800d22ad1fe533e5" - ], - "layout": "IPY_MODEL_2bd15c15e3b748cd9ee001bb86c04b6d" - } - }, - "ef823903b36348978833920bb7e03881": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f07d424e031d457a98a88b94b0c844b8": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f0aa35a704a84967a3471e5f698bb04c": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "initial" - } - }, - "f2c541b4ecd04729a518414f8b878099": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f2e66aebdb4d4812a1f61c1ff990390a": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f3f0b212907843f5be2131b7b60a955d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "", - "description": "Training...: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_6acb415dc8b947bb85ab8f8959bc10e1", - "max": 137, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_b642c546813240a499a165279c743e7a", - "value": 137 - } - }, - "f48e444c929748b2a875c6eecc028bc4": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f5e27446f61747e1a65e477e2975dbcf": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "", - "description": "Training...: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_6669a1844b394ea499dc92e126132bfc", - "max": 137, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_98e148811880459ea7a7641ee3a301b6", - "value": 137 - } - }, - "f7851bd9702949ee8e6278ad69c9d981": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "fc2e1a72620742499457ab2e31da12a9": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "", - "description": "Evaluation...: 100%", - "description_tooltip": null, - "layout": "IPY_MODEL_c7cca560cd4340bb986297655f513746", - "max": 12, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_c55e4e5f6d0a42b4ac25fb6d68a8e842", - "value": 12 - } - }, - "fcf96c9bb04642089466de2421bd1af2": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "fd72a18adfb34bc2b2dab769067dcc1a": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - } - } - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/github-scraping/01_metadata_by_github_api.py b/github-scraping/01_metadata_by_github_api.py deleted file mode 100644 index 162b470..0000000 --- a/github-scraping/01_metadata_by_github_api.py +++ /dev/null @@ -1,91 +0,0 @@ -import requests -import sys - -# Insert GitHub API token here, in place of *TOKEN*. -headers = {"Authorization": "token *TOKEN*"} - -# Constants & language argument. -NUM_REPOS = 10_000 -MIN_STARS = 50 -LANGUAGE = "java" if len(sys.argv) <= 1 else sys.argv[1] - - -def main(): - repositories = set() # Use a set to avoid duplicate entries across pages. - max_stars = 1_000_000_000 # Initialize at a very high value. - while len(repositories) < NUM_REPOS: - new_repositories = run_query(max_stars) - max_stars = min([stars for _, stars in new_repositories]) - # If a query returns no new repositories, drop it. - if len(repositories | new_repositories) == len(repositories): - break - repositories.update(new_repositories) - print(f'Collected {len(repositories):,} repositories so far; lowest number of stars: {max_stars:,}') - - with open(f'{LANGUAGE}-top-repos.txt', 'w') as f: - for repository, _ in sorted(repositories, key=lambda e: e[1], reverse=True): - f.write(f'{repository}\n') - - -def run_query(max_stars): - end_cursor = None # Used to track pagination. - repositories = set() - - while end_cursor != "": - # Extracts non-fork, recently active repositories in the provided language, in groups of 100. - # Leaves placeholders for maximum stars and page cursor. The former allows us to retrieve more than 1,000 repositories - # by repeatedly lowering the bar. - query = f""" - {{ - search(query: "language:{LANGUAGE} fork:false pushed:>2020-01-01 sort:stars stars:<{max_stars}", type: REPOSITORY, first: 100 {', after: "' + end_cursor + '"' if end_cursor else ''}) {{ - edges {{ - node {{ - ... on Repository {{ - url - isPrivate - isDisabled - isLocked - stargazers {{ - totalCount - }} - }} - }} - }} - pageInfo {{ - hasNextPage - endCursor - }} - }} - }} - """ - print(f' Retrieving next page; {len(repositories)} repositories in this batch so far.') - request = requests.post('https://api.github.com/graphql', json={'query': query}, headers=headers) - content = request.json() - end_cursor = get_end_cursor(content) - repositories.update(get_repositories(content)) - if len(repositories) > NUM_REPOS: - break - return repositories - - -def get_end_cursor(content): - page_info = content['data']['search']['pageInfo'] - has_next_page = page_info['hasNextPage'] - if has_next_page: - return page_info['endCursor'] - return "" - - -def get_repositories(content): - edges = content['data']['search']['edges'] - repositories_with_stars = [] - for edge in edges: - if edge['node']['isPrivate'] is False and edge['node']['isDisabled'] is False and edge['node']['isLocked'] is False: - repository = edge['node']['url'] - star_count = edge['node']['stargazers']['totalCount'] - repositories_with_stars.append((repository, star_count)) - return repositories_with_stars - - -if __name__ == '__main__': - main() diff --git a/github-scraping/02_clone_repos.sh b/github-scraping/02_clone_repos.sh deleted file mode 100755 index 66982fe..0000000 --- a/github-scraping/02_clone_repos.sh +++ /dev/null @@ -1,10 +0,0 @@ -in_file=$1 -language=$2 -cat $in_file | xargs -P16 -n1 -I% bash -c 'echo %; \ - name=$(echo % | cut -d"/" -f2); \ - org=$(echo % | cut -d"/" -f1); \ - echo "Cloning $org/$name" - DIR=Repos/'$language'/$org; \ - mkdir -p $DIR; \ - echo $DIR; \ - git clone -q --depth 1 https://github.com/$org/$name $DIR/$name' \ No newline at end of file diff --git a/github-scraping/03_lexer.py b/github-scraping/03_lexer.py deleted file mode 100644 index 1519212..0000000 --- a/github-scraping/03_lexer.py +++ /dev/null @@ -1,50 +0,0 @@ -import os -import sys -import pygments -from pygments.lexers import get_lexer_by_name -from pygments.token import Token - -def main(): - if len(sys.argv) <= 3: - raise ValueError('Provide at least an input directory, an output directory, and a language.') - input_directory = sys.argv[1] - output_directory = sys.argv[2] - language = sys.argv[3] - if input_directory.endswith('/'): - input_directory = input_directory[:-1] - if not os.path.exists(output_directory): - os.makedirs(output_directory) - - lexer = get_lexer_by_name(language) - language_extensions = set(ext.lower()[1:] for ext in lexer.filenames) - - with open(os.path.join(output_directory, os.path.basename(input_directory) + '.txt'), 'w') as f_out: - for root, _, files in os.walk(input_directory): - for name in files: - ext = os.path.splitext(name)[1].lower() - if ext in language_extensions: - print(f'Lexing: {root}, {name}') - lex_file(os.path.join(root, name), f_out, lexer) - - -def lex_file(file_path, f_out, lexer): - with open(file_path, errors='ignore') as f_in: - text = f_in.read() - - lexed = [] - for (ttype, token) in pygments.lex(text, lexer): - if ttype in Token.Text: - continue - elif ttype in Token.Comment: - continue - else: - lexed.append(token.replace('\t', '#TAB#')) - - # Skip empty files. - if not lexed: - return - f_out.write('\t'.join(lexed)) - f_out.write('\n') - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/github-scraping/README.md b/github-scraping/README.md deleted file mode 100644 index aeb88ff..0000000 --- a/github-scraping/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Github Scraping Scripts - -This directory contains scripts to scrape github (based on scripts originally by [Vincent Hellendoorn](https://vhellendoorn.github.io/)). - -* `01_metadata_by_github_api.py`: A script to crawl repo metadata using the github API. Currently it crawls N repositories of down to M github stars for language L. -* `02_clone_repos.sh`: Actually download the repos. -* `03_lexer.py`: Finds and lexes code by file extension. Note that this does some stuff that might be bad for model training such as getting rid of whitespace, so it might need to be fixed. diff --git a/gpt-neo-test.ipynb b/gpt-neo-test.ipynb deleted file mode 100644 index 90e806c..0000000 --- a/gpt-neo-test.ipynb +++ /dev/null @@ -1,487 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 5, - "id": "84b1a438-cf1d-402e-a56f-2c4f9dd5ad51", - "metadata": {}, - "outputs": [], - "source": [ - "from transformers import AutoTokenizer, FlaxGPTNeoForCausalLM, AutoModelForMaskedLM" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "7d50cd18-33ed-4b67-82ad-5c48eb9a9b36", - "metadata": {}, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "# model_ckpt = 'EleutherAI/gpt-neo-125M'\n", - "model_ckpt = (Path.home()/'gpt-neo-125M-code-clippy').as_posix()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2ec0c4cc-a1bc-4dda-bd0b-72891b519b39", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "065c03c3-2e4a-4f20-a30d-25ada1418b18", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:absl:Starting the local TPU driver.\n", - "INFO:absl:Unable to initialize backend 'tpu_driver': Not found: Unable to find driver in registry given worker: local://\n", - "INFO:absl:Unable to initialize backend 'gpu': Not found: Could not find registered platform with name: \"cuda\". Available platform names are: TPU Interpreter Host\n" - ] - } - ], - "source": [ - "tokenizer = AutoTokenizer.from_pretrained(model_ckpt)\n", - "model = FlaxGPTNeoForCausalLM.from_pretrained(model_ckpt)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "e2f9fb26-2e26-4f57-aa93-e349475203f3", - "metadata": {}, - "outputs": [], - "source": [ - "tokenizer.pad_token = tokenizer.eos_token" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "id": "75c0c2f6-47ad-41c3-8c66-a1ceeecde061", - "metadata": {}, - "outputs": [], - "source": [ - "prompt = \"\"\"\n", - "import numpy as np\n", - "import pandas as pd\n", - "import matplotlib.pyplot as plt\n", - "\n", - "x = np.random.randn(10, 10)\n", - "\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "id": "666977a1-de0d-4900-bf61-ae2b672e51bc", - "metadata": {}, - "outputs": [], - "source": [ - "inputs = tokenizer(prompt, return_tensors='jax')\n", - "input_ids = inputs.input_ids" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "id": "249e4a8a-7a7e-4e8b-83be-7184a4c0dd0b", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(1, 40, 50257)" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "outputs = model(**inputs)\n", - "outputs.logits.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "id": "eee873f5-073c-4cbe-8b15-114ea18b2de8", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "DeviceArray([[ 198, 11748, 299, 32152, 355, 45941, 198, 11748,\n", - " 19798, 292, 355, 279, 67, 198, 11748, 2603,\n", - " 29487, 8019, 13, 9078, 29487, 355, 458, 83,\n", - " 198, 198, 87, 796, 45941, 13, 25120, 13,\n", - " 25192, 77, 7, 940, 11, 838, 8, 198]], dtype=int32)" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "input_ids" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "id": "82666225-3ab7-405f-9536-4e9e3085be24", - "metadata": {}, - "outputs": [], - "source": [ - "out = model.generate(input_ids,\n", - " max_length=200, \n", - " num_beams=1,\n", - " pad_token_id = tokenizer.pad_token_id\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "id": "c6cc862b-23ef-417d-ae83-1b2eafb0460f", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "FlaxGreedySearchOutput(sequences=DeviceArray([[ 198, 11748, 299, 32152, 355, 45941, 198, 11748,\n", - " 19798, 292, 355, 279, 67, 198, 11748, 2603,\n", - " 29487, 8019, 13, 9078, 29487, 355, 458, 83,\n", - " 198, 198, 87, 796, 45941, 13, 25120, 13,\n", - " 25192, 77, 7, 940, 11, 838, 8, 198,\n", - " 88, 796, 45941, 13, 25120, 13, 25192, 77,\n", - " 7, 940, 11, 838, 8, 198, 198, 2,\n", - " 220, 220, 220, 220, 220, 220, 220, 220,\n", - " 220, 220, 220, 220, 220, 220, 220, 220,\n", - " 220, 220, 220, 220, 220, 220, 220, 220,\n", - " 220, 220, 220, 220, 220, 220, 220, 220,\n", - " 220, 220, 220, 220, 220, 220, 220, 220,\n", - " 220, 220, 220, 220, 220, 220, 220, 220,\n", - " 220, 220, 220, 220, 220, 220, 220, 220,\n", - " 220, 220, 220, 220, 220, 220, 220, 220,\n", - " 220, 220, 220, 220, 220, 220, 220, 220,\n", - " 220, 220, 220, 220, 220, 220, 220, 220,\n", - " 220, 220, 220, 220, 220, 220, 220, 220,\n", - " 220, 220, 220, 220, 220, 220, 220, 220,\n", - " 220, 220, 220, 220, 220, 220, 220, 220,\n", - " 220, 220, 220, 220, 220, 220, 220, 220,\n", - " 220, 220, 220, 220, 220, 220, 220, 220,\n", - " 220, 220, 220, 220, 220, 220, 220, 220,\n", - " 220, 220, 220, 220, 220, 220, 220, 220,\n", - " 220, 220, 220, 220, 220, 220, 220, 220]], dtype=int32))" - ] - }, - "execution_count": 38, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "out" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "id": "8f6c746a-2d56-4da4-acb5-e066a6a230f2", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "import matplotlib.pyplot as plt\n", - "\n", - "x = np.random.randn(10, 10)\n", - "y = np.random.randn(10, 10)\n", - "\n", - "# \n" - ] - } - ], - "source": [ - "print(tokenizer.decode(out[0][0]))" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "id": "b6effeaa-2237-47bc-b0f6-940c4e274c38", - "metadata": {}, - "outputs": [], - "source": [ - "from transformers import GPTNeoForCausalLM, AutoModelForCausalLM" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "id": "3665a3fd-5d92-45e8-8fde-393ec803383a", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/arto/transformers/src/transformers/modeling_flax_pytorch_utils.py:201: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at /pytorch/torch/csrc/utils/tensor_numpy.cpp:180.)\n", - " pt_model_dict[flax_key] = torch.from_numpy(flax_tensor)\n", - "All Flax model weights were used when initializing GPTNeoForCausalLM.\n", - "\n", - "Some weights of GPTNeoForCausalLM were not initialized from the Flax model and are newly initialized: ['lm_head.weight', 'transformer.h.1.attn.attention.masked_bias', 'transformer.h.6.attn.attention.bias', 'transformer.h.7.attn.attention.masked_bias', 'transformer.h.10.attn.attention.masked_bias', 'transformer.h.4.attn.attention.bias', 'transformer.h.2.attn.attention.bias', 'transformer.h.6.attn.attention.masked_bias', 'transformer.h.2.attn.attention.masked_bias', 'transformer.h.0.attn.attention.bias', 'transformer.h.3.attn.attention.masked_bias', 'transformer.h.5.attn.attention.masked_bias', 'transformer.h.4.attn.attention.masked_bias', 'transformer.h.8.attn.attention.masked_bias', 'transformer.h.11.attn.attention.masked_bias', 'transformer.h.9.attn.attention.masked_bias', 'transformer.h.0.attn.attention.masked_bias', 'transformer.h.8.attn.attention.bias', 'transformer.h.10.attn.attention.bias']\n", - "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" - ] - } - ], - "source": [ - "model = GPTNeoForCausalLM.from_pretrained(model_ckpt, from_flax=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5fa23301-6f5d-40d5-b614-f14330df894a", - "metadata": {}, - "outputs": [], - "source": [ - "from transormers import AutoModelForMaskedLM\n", - "model = AutoModelForMaskedLM.from_pretrained(model_ckpt, from_flax=True)\n", - "model.save_pretrained(model_ckpt, save_config=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "id": "35114633-bb5f-4c00-ae16-540a7fabb126", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "All Flax model weights were used when initializing GPTNeoForCausalLM.\n", - "\n", - "Some weights of GPTNeoForCausalLM were not initialized from the Flax model and are newly initialized: ['lm_head.weight', 'transformer.h.1.attn.attention.masked_bias', 'transformer.h.6.attn.attention.bias', 'transformer.h.7.attn.attention.masked_bias', 'transformer.h.10.attn.attention.masked_bias', 'transformer.h.4.attn.attention.bias', 'transformer.h.2.attn.attention.bias', 'transformer.h.6.attn.attention.masked_bias', 'transformer.h.2.attn.attention.masked_bias', 'transformer.h.0.attn.attention.bias', 'transformer.h.3.attn.attention.masked_bias', 'transformer.h.5.attn.attention.masked_bias', 'transformer.h.4.attn.attention.masked_bias', 'transformer.h.8.attn.attention.masked_bias', 'transformer.h.11.attn.attention.masked_bias', 'transformer.h.9.attn.attention.masked_bias', 'transformer.h.0.attn.attention.masked_bias', 'transformer.h.8.attn.attention.bias', 'transformer.h.10.attn.attention.bias']\n", - "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" - ] - } - ], - "source": [ - "model = AutoModelForCausalLM.from_pretrained(model_ckpt, from_flax=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "id": "15cd3853-1308-46e1-90c1-52b3af0fcac4", - "metadata": {}, - "outputs": [], - "source": [ - "prompt = \"\"\"\n", - "my_list = ['banana', 'apple', 'orange', 'pineapple']\n", - "\n", - "#Using brute force method\n", - "last_element = my_list[len(my_list) - 1]\n", - "\n", - "#Using negative indeces\n", - "last_element = my_list[-1]\n", - "\n", - "#Using pop method\n", - "\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 72, - "id": "2f2fc69a-f8f5-4859-bb2e-5c33e63f064a", - "metadata": {}, - "outputs": [], - "source": [ - "prompt = \"\"\"\n", - "def get_vowels(string):\n", - " return [vowel for vowel in string if vowel in 'aeiou'] \n", - "\n", - "print(\"Vowels are:\",\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 77, - "id": "517aa451-3316-45fc-97ab-1a9a52ba55b6", - "metadata": {}, - "outputs": [], - "source": [ - "prompt = \"\"\"import time\n", - "\n", - "start_time = time.time()\n", - "\n", - "total = 0\n", - "for i in range(10):\n", - " total += i\n", - "print(\"Sum:\", total)\n", - "\n", - "end_time = time.time()\n", - "time_taken = \"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 78, - "id": "749f6c3d-e1a4-4df7-be81-086024345766", - "metadata": {}, - "outputs": [], - "source": [ - "inputs = tokenizer(prompt, return_tensors='pt')\n", - "input_ids = inputs.input_ids" - ] - }, - { - "cell_type": "code", - "execution_count": 81, - "id": "6f60e3f0-d051-4df1-8258-7bc479486603", - "metadata": {}, - "outputs": [], - "source": [ - "out = model.generate(input_ids,\n", - " max_length=200, \n", - " num_beams=1,\n", - " pad_token_id = tokenizer.pad_token_id\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 82, - "id": "9d17aec3-e42a-43d6-a535-2eeaad2a9c78", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "import time\n", - "\n", - "start_time = time.time()\n", - "\n", - "total = 0\n", - "for i in range(10):\n", - " total += i\n", - "print(\"Sum:\", total)\n", - "\n", - "end_time = time.time()\n", - "time_taken = time.time()\n", - "\n", - "# \n" - ] - } - ], - "source": [ - "print(tokenizer.decode(out[0]))" - ] - }, - { - "cell_type": "code", - "execution_count": 76, - "id": "57574549-bd1d-46b0-98ca-352662f735d2", - "metadata": {}, - "outputs": [], - "source": [ - "model.save_pretrained(model_ckpt, save_config=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "0578148d-497d-422f-b7fb-b644d2a7c62f", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2021-07-06 15:02:08.590730: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n", - "2021-07-06 15:02:08.590769: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n" - ] - } - ], - "source": [ - "from transformers import TrainingArguments" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "63ffd8ff-8e95-4aad-9068-b27fd8c129bb", - "metadata": {}, - "outputs": [], - "source": [ - "from dataclasses import fields" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "a9a89020-e7b0-4826-88e6-8ac4f4c6f89e", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Field(name='skip_memory_metrics',type=,default=True,default_factory=,init=True,repr=True,hash=None,compare=True,metadata=mappingproxy({'help': 'Whether or not to skip adding of memory profiler reports to metrics.'}),_field_type=_FIELD)\n" - ] - } - ], - "source": [ - "for f in fields(TrainingArguments):\n", - " if f.name == \"skip_memory_metrics\":\n", - " print(f)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "beda87b7-c461-4f92-8988-4255a8e79cf9", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/requirements.txt b/requirements_scripts.txt similarity index 100% rename from scripts/requirements.txt rename to requirements_scripts.txt diff --git a/run_clm_gpt_neo.sh b/run_clm_gpt_neo.sh deleted file mode 100644 index 6887341..0000000 --- a/run_clm_gpt_neo.sh +++ /dev/null @@ -1,32 +0,0 @@ -#! /bin/bash -./run_clm_flax.py \ - --output_dir $HOME/tmp/gpt-neo-125M-code-clippy \ - --model_name_or_path="EleutherAI/gpt-neo-125M" \ - --dataset_name="code_search_net" \ - --dataset_config_name="python" \ - --text_column_name="func_code_string" \ - --do_train --do_eval \ - --block_size="2048" \ - --per_device_train_batch_size="8" \ - --per_device_eval_batch_size="16" \ - --preprocessing_num_workers="8" \ - --learning_rate="6e-4" \ - --adafactor \ - --warmup_steps="100" \ - --adam_beta1="0.9" \ - --adam_beta2="0.98" \ - --weight_decay="0.01" \ - --overwrite_output_dir \ - --num_train_epochs="10" \ - --logging_steps="100" \ - --eval_steps="200" \ - --push_to_hub="False" \ - --report_to="none" \ - --dtype="bfloat16" \ - --skip_memory_metrics="False" \ - --save_steps="200" \ - --save_total_limit 2 \ - --gradient_accumulation_steps 8 \ - # --resume_from_checkpoint $HOME/gpt-neo-125M-code-clippy/ckpt_201 \ - # --max_train_samples="10000" \ - # --max_eval_samples="1000" diff --git a/run_clm_streaming.sh b/run_clm_streaming.sh deleted file mode 100644 index b0cf84b..0000000 --- a/run_clm_streaming.sh +++ /dev/null @@ -1,37 +0,0 @@ -#! /bin/bash -./run_clm_streaming_flax_v2.py \ - --output_dir $HOME/gpt-neo-125M-test \ - --model_name_or_path="EleutherAI/gpt-neo-125M" \ - --dataset_name $HOME/gpt-code-clippy/code_clippy.py \ - --data_dir /home/shared/code-clippy-dataset/merged-data \ - --text_column_name="text" \ - --do_train --do_eval \ - --block_size="2048" \ - --per_device_train_batch_size="8" \ - --per_device_eval_batch_size="16" \ - --preprocessing_num_workers="8" \ - --learning_rate="6e-4" \ - --max_steps 500 \ - --warmup_steps 150 \ - --decay_steps 250 \ - --adam_beta1="0.9" \ - --adam_beta2="0.95" \ - --weight_decay="0.01" \ - --overwrite_output_dir \ - --logging_steps="10" \ - --eval_steps="50" \ - --push_to_hub="True" \ - --report_to="all" \ - --dtype="bfloat16" \ - --skip_memory_metrics="False" \ - --save_steps="50" \ - --save_total_limit 2 \ - --gradient_accumulation_steps 8 \ - --report_to="wandb" \ - --run_name="testing-mini" \ - --max_eval_samples 100 \ - --save_optimizer true \ - # --adafactor \ - # --resume_from_checkpoint $HOME/gpt-neo-125M-code-clippy/ \ - # --max_train_samples="10000" \ - diff --git a/run_clm_streaming_flax.py b/run_clm_streaming_flax.py deleted file mode 100755 index d3d9adf..0000000 --- a/run_clm_streaming_flax.py +++ /dev/null @@ -1,885 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -# Copyright 2021 The HuggingFace Team All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Pre-training/Fine-tuning the library models for causal language modeling (GPT, GPT-2, CTRL, ...) on a text file or a dataset. - -Here is the full list of checkpoints on the hub that can be fine-tuned by this script: -https://huggingface.co/models?filter=causal-lm -""" -# You can also adapt this script on your own causal language modeling task. Pointers for this are left as comments. - -from ast import Str -import logging -import math -import os -import sys -import time -from dataclasses import dataclass, field -from pathlib import Path -from typing import Callable, Optional -import json -import shutil -from collections import defaultdict -import numpy as np -# from queue import Queue -# import threading -from multiprocessing import Process, Queue -import datasets -from datasets import Dataset, load_dataset -from tqdm import tqdm - -import jax -import jax.profiler -import jax.numpy as jnp -import optax -import transformers -from flax import jax_utils, traverse_util -from flax.jax_utils import unreplicate -from flax.training import train_state -from flax.training.common_utils import get_metrics, onehot, shard, shard_prng_key -from flax.serialization import to_bytes, from_bytes -from transformers import ( - CONFIG_MAPPING, - FLAX_MODEL_FOR_CAUSAL_LM_MAPPING, - AutoConfig, - AutoTokenizer, - FlaxAutoModelForCausalLM, - HfArgumentParser, - TrainingArguments, - is_tensorboard_available, -) -from transformers.testing_utils import CaptureLogger - -from importlib.util import find_spec - -logger = logging.getLogger(__name__) - -MODEL_CONFIG_CLASSES = list(FLAX_MODEL_FOR_CAUSAL_LM_MAPPING.keys()) -MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES) - - -@dataclass -class ModelArguments: - """ - Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch. - """ - - model_name_or_path: Optional[str] = field( - default=None, - metadata={ - "help": "The model checkpoint for weights initialization." - "Don't set if you want to train a model from scratch." - }, - ) - model_type: Optional[str] = field( - default=None, - metadata={"help": "If training from scratch, pass a model type from the list: " + ", ".join(MODEL_TYPES)}, - ) - config_name: Optional[str] = field( - default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"} - ) - tokenizer_name: Optional[str] = field( - default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"} - ) - cache_dir: Optional[str] = field( - default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"} - ) - use_fast_tokenizer: bool = field( - default=True, - metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."}, - ) - dtype: Optional[str] = field( - default="float32", - metadata={ - "help": "Floating-point format in which the model weights should be initialized and trained. Choose one of `[float32, float16, bfloat16]`." - }, - ) - - -@dataclass -class DataTrainingArguments: - """ - Arguments pertaining to what data we are going to input our model for training and eval. - """ - - dataset_name: Optional[str] = field( - default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."} - ) - dataset_config_name: Optional[str] = field( - default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."} - ) - train_file: Optional[str] = field(default=None, metadata={"help": "The input training data file (a text file)."}) - validation_file: Optional[str] = field( - default=None, - metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."}, - ) - data_dir: Optional[str] = field(default=None, metadata={"help": "Path to data directory."}) - max_train_samples: Optional[int] = field( - default=None, - metadata={ - "help": "For debugging purposes or quicker training, truncate the number of training examples to this " - "value if set." - }, - ) - max_eval_samples: Optional[int] = field( - default=None, - metadata={ - "help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this " - "value if set." - }, - ) - overwrite_cache: bool = field( - default=False, metadata={"help": "Overwrite the cached training and evaluation sets"} - ) - validation_split_percentage: Optional[int] = field( - default=5, - metadata={ - "help": "The percentage of the train set used as validation set in case there's no validation split" - }, - ) - block_size: Optional[int] = field( - default=None, - metadata={ - "help": "Optional input sequence length after tokenization. " - "The training dataset will be truncated in block of this size for training. " - "Default to the model max input length for single sentence inputs (take into account special tokens)." - }, - ) - overwrite_cache: bool = field( - default=False, metadata={"help": "Overwrite the cached training and evaluation sets"} - ) - preprocessing_num_workers: Optional[int] = field( - default=None, - metadata={"help": "The number of processes to use for the preprocessing."}, - ) - text_column_name: Optional[str] = field( - default='text', - metadata={"help": "Column containing main text data."}, - ) - shuffle_buffer_size: int = field( - default=10000, metadata={"help": "The number of examples to pre-load for shuffling."} - ) - num_train_steps: int = field(default=50000, metadata={"help": "The number of training steps."}) - num_eval_samples: int = field(default=50000, metadata={"help": "The number of samples to be used for evaluation"}) - prefetch_buffer: int = field(default=8, metadata={"help": "The number of batches to prefetch for loading"}) - - def __post_init__(self): - if self.dataset_name is None and self.train_file is None and self.validation_file is None: - raise ValueError("Need either a dataset name or a training/validation file.") - else: - if self.train_file is not None: - extension = self.train_file.split(".")[-1] - assert extension in ["csv", "json", "txt"], "`train_file` should be a csv, a json or a txt file." - if self.validation_file is not None: - extension = self.validation_file.split(".")[-1] - assert extension in ["csv", "json", "txt"], "`validation_file` should be a csv, a json or a txt file." - - -class TrainState(train_state.TrainState): - dropout_rng: jnp.ndarray - - def replicate(self): - return jax_utils.replicate(self).replace(dropout_rng=shard_prng_key(self.dropout_rng)) - - -def generate_batch_splits(samples_idx: jnp.ndarray, batch_size: int) -> jnp.ndarray: - num_samples = len(samples_idx) - samples_to_remove = num_samples % batch_size - - if samples_to_remove != 0: - samples_idx = samples_idx[:-samples_to_remove] - sections_split = num_samples // batch_size - batch_idx = np.split(samples_idx, sections_split) - return batch_idx - - -def advance_iter_and_group_samples(train_iterator, num_samples, max_seq_length): - """ - The training iterator is advanced so that after groupifying the samples, - `num_samples` of length `max_seq_length` are returned. - """ - num_total_tokens = max_seq_length * num_samples - samples = defaultdict(list) - - i = 0 - while i < num_total_tokens: - tokenized_samples = next(train_iterator) - i += len(tokenized_samples["input_ids"]) - - # concatenate tokenized samples to list - samples = {k: samples[k] + tokenized_samples[k] for k in tokenized_samples.keys()} - - # Concatenated tokens are split to lists of length `max_seq_length`. - # Note that remainedr of % max_seq_length are thrown away. - def group_texts(examples): - result = { - k: [t[i : i + max_seq_length] for i in range(0, num_total_tokens, max_seq_length)] - for k, t in examples.items() - } - return result - - grouped_samples = group_texts(samples) - return grouped_samples - -def make_batch(samples): - batch = {k:jnp.array(v) for k,v in samples.items()} - batch['labels'] = batch['input_ids'].copy() - return batch - -# class PrefetchDataloader(threading.Thread): -# "Prefetch dataloader for IterableDataset" -# def __init__(self, dataset, batch_size, sequence_length, prefetch_buffer=1, shuffle=True, shuffle_buffer=1000, seed=0): -# super().__init__(daemon=True) -# self.bs = batch_size -# self.seq_len = sequence_length -# self.max_length = batch_size * sequence_length -# self.prefetch_buffer = prefetch_buffer -# self.shuffle = shuffle -# self.shuffle_buffer = shuffle_buffer -# self.seed = seed -# self.dataset = dataset -# if shuffle: -# shuffled_dataset = dataset.shuffle(shuffle_buffer, seed=self.seed) -# self.seed += 1 -# self.ds_iter = iter(shuffled_dataset) -# else: -# self.ds_iter = iter(dataset) -# self.queue = Queue(prefetch_buffer) -# self.rem = defaultdict(list) -# self.start() - -# def __next__(self): -# batch = self.queue.get() -# return batch - -# def run(self): -# while True: -# # prepair next batch -# sample = self.rem.copy() -# l = len(sample["input_ids"]) -# max_length = self.max_length -# while l < max_length: -# next_sample = next(self.ds_iter) -# l += len(next_sample["input_ids"]) -# sample = {k:sample[k]+next_sample[k] for k in next_sample.keys()} - -# self.rem = {k:v[max_length:] for k,v in sample.items()} -# sample = {k:v[:max_length] for k,v in sample.items()} -# # regroup to shape [bs x seq_len] -# samples = {k:np.array([v[i*self.seq_len:(i+1)*self.seq_len] for i in range(self.bs)]) for k,v in sample.items()} - -# self.queue.put(make_batch(samples)) - -# def __iter__(self): -# return self - - -class PrefetchDataloader(Process): - "Prefetch dataloader for IterableDataset" - def __init__(self, dataset, batch_size, sequence_length, prefetch_buffer=1, shuffle=True, shuffle_buffer=1000, seed=0): - super().__init__(daemon=True) - self.bs = batch_size - self.seq_len = sequence_length - self.max_length = batch_size * sequence_length - self.prefetch_buffer = prefetch_buffer - self.shuffle = shuffle - self.shuffle_buffer = shuffle_buffer - self.seed = seed - self.dataset = dataset - if shuffle: - shuffled_dataset = dataset.shuffle(shuffle_buffer, seed=self.seed) - self.seed += 1 - self.ds_iter = iter(shuffled_dataset) - else: - self.ds_iter = iter(dataset) - self.queue = Queue(prefetch_buffer) - self.rem = defaultdict(list) - self.start() - - def __next__(self): - return make_batch(self.queue.get()) - - def run(self): - while True: - # prepair next batch - sample = self.rem.copy() - l = len(sample["input_ids"]) - max_length = self.max_length - while l < max_length: - next_sample = next(self.ds_iter) - l += len(next_sample["input_ids"]) - sample = {k:sample[k]+next_sample[k] for k in next_sample.keys()} - - self.rem = {k:v[max_length:] for k,v in sample.items()} - sample = {k:v[:max_length] for k,v in sample.items()} - # regroup to shape [bs x seq_len] - samples = {k:np.array([v[i*self.seq_len:(i+1)*self.seq_len] for i in range(self.bs)]) for k,v in sample.items()} - - self.queue.put(samples) - - def __iter__(self): - return self - -def data_loader(rng: jax.random.PRNGKey, dataset: Dataset, batch_size: int, shuffle: bool = False): - """ - Returns batches of size `batch_size` from truncated `dataset`, sharded over all local devices. - Shuffle batches if `shuffle` is `True`. - """ - steps_per_epoch = len(dataset) // batch_size - - if shuffle: - batch_idx = jax.random.permutation(rng, len(dataset)) - else: - batch_idx = jnp.arange(len(dataset)) - - batch_idx = batch_idx[: steps_per_epoch * batch_size] # Skip incomplete batch. - batch_idx = batch_idx.reshape((steps_per_epoch, batch_size)) - - for idx in batch_idx: - batch = dataset[idx] - batch = {k: jnp.array(v) for k, v in batch.items()} - - batch = shard(batch) - - yield batch - - -def write_train_metric(summary_writer, train_metrics, train_time, step): - summary_writer.scalar("train_time", train_time, step) - - train_metrics = get_metrics(train_metrics) - for key, vals in train_metrics.items(): - tag = f"train_{key}" - for i, val in enumerate(vals): - summary_writer.scalar(tag, val, step - len(vals) + i + 1) - -def write_eval_metric(summary_writer, eval_metrics, step): - for metric_name, value in eval_metrics.items(): - summary_writer.scalar(f"eval_{metric_name}", value, step) - - -def create_learning_rate_fn( - num_train_steps: int, train_batch_size: int, num_warmup_steps: int, learning_rate: float -) -> Callable[[int], jnp.array]: - """Returns a linear warmup, linear_decay learning rate function.""" - warmup_fn = optax.linear_schedule(init_value=0.0, end_value=learning_rate, transition_steps=num_warmup_steps) - decay_fn = optax.linear_schedule( - init_value=learning_rate, end_value=0, transition_steps=num_train_steps - num_warmup_steps - ) - schedule_fn = optax.join_schedules(schedules=[warmup_fn, decay_fn], boundaries=[num_warmup_steps]) - return schedule_fn - -# utils -def mb_item(x): - return x.item() if hasattr(x, "item") else x - -#checkpoint functions -def save_checkpoint(model, save_dir, state, with_opt:bool=True, push_to_hub:bool=False): - state = jax_utils.unreplicate(state) - logger.info(f"SAVING CHECKPOINT IN {save_dir}...") - save_dir = f"{save_dir}/ckpt-{mb_item(state.step)-1}" - model.save_pretrained( - save_dir, - params=state.params, - push_to_hub=push_to_hub, - commit_message=f"Saving weights and logs at step {mb_item(state.step)-1}", - ) - if with_opt: - with open(os.path.join(save_dir, "opt_state.msgpack"), "wb") as f: - f.write(to_bytes(state.opt_state)) - with open(os.path.join(save_dir, "training_state.json"), "w") as f: - json.dump({"step": state.step.item()}, f) - logger.info("checkpoint saved") - -def restore_checkpoint(save_dir, state): - logger.info(f"RESTORING CHECKPOINT FROM {save_dir}...") - with open(os.path.join(save_dir, "flax_model.msgpack"), "rb") as f: - params = from_bytes(state.params, f.read()) - - with open(os.path.join(save_dir, "opt_state.msgpack"), "rb") as f: - opt_state = from_bytes(state.opt_state, f.read()) - - with open(os.path.join(save_dir, "training_state.json"), "r") as f: - training_state = json.load(f) - step = training_state["step"] - - logger.info("checkpoint restored") - return state.replace(step=step, params=params, opt_state=opt_state), step - -def rotate_checkpoints(ckpt_dir:str, save_total_limit:int): - "Removes older checkpoints so that `save_total_limit` checkpoints are kept" - # TODO: what to remove is decided using step number only, we might want to improve that - ckpts = [str(x) for x in Path(ckpt_dir).glob("ckpt-*")] - # sort checkpoints by step - ckpts_sorted = sorted(ckpts, key=lambda x: int(x.split('-')[-1])) - ckpts_to_delete = ckpts_sorted[:-save_total_limit] - for ckpt in ckpts_to_delete: - logger.info(f"Deleting older checkpoint [{ckpt}] due to save_total_limit ({save_total_limit})") - shutil.rmtree(ckpt) - -def main(): - # See all possible arguments in src/transformers/training_args.py - # or by passing the --help flag to this script. - # We now keep distinct sets of args, for a cleaner separation of concerns. - - parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)) - if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): - # If we pass only one argument to the script and it's the path to a json file, - # let's parse it to get our arguments. - model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1])) - else: - model_args, data_args, training_args = parser.parse_args_into_dataclasses() - - if ( - os.path.exists(training_args.output_dir) - and os.listdir(training_args.output_dir) - and training_args.do_train - and not training_args.overwrite_output_dir - ): - raise ValueError( - f"Output directory ({training_args.output_dir}) already exists and is not empty." - "Use --overwrite_output_dir to overcome." - ) - - # Make one log on every process with the configuration for debugging. - logging.basicConfig( - format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", - datefmt="%m/%d/%Y %H:%M:%S", - level=logging.INFO, - ) - # Setup logging, we only want one process per machine to log things on the screen. - logger.setLevel(logging.INFO if jax.process_index() == 0 else logging.ERROR) - if jax.process_index() == 0: - datasets.utils.logging.set_verbosity_warning() - transformers.utils.logging.set_verbosity_info() - else: - datasets.utils.logging.set_verbosity_error() - transformers.utils.logging.set_verbosity_error() - - # Set the verbosity to info of the Transformers logger (on main process only): - logger.info(f"Training/evaluation parameters {training_args}") - - # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below) - # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/ - # (the dataset will be downloaded automatically from the datasets Hub). - # - # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called - # 'text' is found. You can easily tweak this behavior (see below). - # - # In distributed training, the load_dataset function guarantees that only one local process can concurrently - # download the dataset. - if data_args.dataset_name is not None: - # Downloading and loading a dataset from the hub. - train_dataset = load_dataset( - data_args.dataset_name, - data_dir=data_args.data_dir, - cache_dir=model_args.cache_dir, - streaming=True, - split="train" - ) - eval_dataset = load_dataset( - data_args.dataset_name, - data_dir=data_args.data_dir, - cache_dir=model_args.cache_dir, - split="validation" - ) - - # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at - # https://huggingface.co/docs/datasets/loading_datasets.html. - - # Load pretrained model and tokenizer - - # Distributed training: - # The .from_pretrained methods guarantee that only one local process can concurrently - # download model & vocab. - if model_args.config_name: - config = AutoConfig.from_pretrained(model_args.config_name, cache_dir=model_args.cache_dir) - elif model_args.model_name_or_path: - config = AutoConfig.from_pretrained(model_args.model_name_or_path, cache_dir=model_args.cache_dir) - else: - config = CONFIG_MAPPING[model_args.model_type]() - logger.warning("You are instantiating a new config instance from scratch.") - - if model_args.tokenizer_name: - tokenizer = AutoTokenizer.from_pretrained( - model_args.tokenizer_name, cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer - ) - elif model_args.model_name_or_path: - tokenizer = AutoTokenizer.from_pretrained( - model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_fast=model_args.use_fast_tokenizer - ) - else: - raise ValueError( - "You are instantiating a new tokenizer from scratch. This is not supported by this script." - "You can do it from another script, save it, and load it from here, using --tokenizer_name." - ) - - if model_args.model_name_or_path: - model = FlaxAutoModelForCausalLM.from_pretrained( - model_args.model_name_or_path, config=config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype) - ) - else: - model = FlaxAutoModelForCausalLM.from_config( - config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype) - ) - - # Preprocessing the datasets. - # First we tokenize all the texts. - column_names = eval_dataset.column_names - text_column_name = data_args.text_column_name if data_args.text_column_name in column_names else column_names[0] - - # since this will be pickled to avoid _LazyModule error in Hasher force logger loading before tokenize_function - tok_logger = transformers.utils.logging.get_logger("transformers.tokenization_utils_base") - - def tokenize_function(examples): - with CaptureLogger(tok_logger) as cl: - output = tokenizer(examples[text_column_name]) - # clm input could be much much longer than block_size - if "Token indices sequence length is longer than the" in cl.out: - tok_logger.warning( - "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model." - ) - return output - - tokenized_dataset = train_dataset.map( - tokenize_function, - batched=True, - ) - tokenized_eval_dataset = eval_dataset.map( - tokenize_function, - batched=True, - remove_columns=column_names, - num_proc=data_args.preprocessing_num_workers, - load_from_cache_file=not data_args.overwrite_cache, - ) - - if data_args.block_size is None: - block_size = tokenizer.model_max_length - if block_size > config.max_position_embeddings: - logger.warning( - f"The tokenizer picked seems to have a very large `model_max_length` ({tokenizer.model_max_length}). " - "Picking 1024 instead. You can change that default value by passing --block_size xxx." - ) - block_size = 1024 - else: - if data_args.block_size > tokenizer.model_max_length: - logger.warning( - f"The block_size passed ({data_args.block_size}) is larger than the maximum length for the model" - f"({tokenizer.model_max_length}). Using block_size={tokenizer.model_max_length}." - ) - block_size = min(data_args.block_size, tokenizer.model_max_length) - - # # Main data processing function that will concatenate all texts from our dataset and generate chunks of block_size. - def group_texts(examples): - # Concatenate all texts. - concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()} - total_length = len(concatenated_examples[list(examples.keys())[0]]) - # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can - # customize this part to your needs. - total_length = (total_length // block_size) * block_size - # Split by chunks of max_len. - result = { - k: [t[i : i + block_size] for i in range(0, total_length, block_size)] - for k, t in concatenated_examples.items() - } - result["labels"] = result["input_ids"].copy() - return result - - # Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a remainder - # for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value might be slower - # to preprocess. - # - # To speed up this part, we use multiprocessing. See the documentation of the map method for more information: - # https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map - - shuffle_seed = training_args.seed - # if training_args.do_train: - # if "train" not in tokenized_dataset: - # raise ValueError("--do_train requires a train dataset") - # train_dataset = tokenized_dataset - # if data_args.max_train_samples is not None: - # train_dataset = train_dataset.take(range(data_args.max_train_samples)) - # train_dataset = train_dataset.shuffle(buffer_size=data_args.shuffle_buffer_size, seed=shuffle_seed) - # train_iter = iter(train_dataset) - - - # Store some constant - num_epochs = int(training_args.num_train_epochs) - train_batch_size = int(training_args.per_device_train_batch_size) * jax.device_count() * training_args.gradient_accumulation_steps - eval_batch_size = int(training_args.per_device_eval_batch_size) * jax.device_count() - # steps_per_epoch = len(train_dataset) // train_batch_size - total_train_steps = training_args.max_steps - - train_dl = PrefetchDataloader( - tokenized_dataset, - int(training_args.per_device_train_batch_size) * jax.device_count(), - block_size, - prefetch_buffer=data_args.prefetch_buffer, - seed=shuffle_seed - ) - # evaluation data is not in streaming mode - if training_args.do_eval: - eval_dataset = tokenized_eval_dataset.map( - group_texts, - batched=True, - num_proc=data_args.preprocessing_num_workers, - load_from_cache_file=not data_args.overwrite_cache, - ) - if data_args.max_eval_samples is not None: - eval_dataset = eval_dataset.select(range(data_args.max_eval_samples)) - - # Enable tensorboard only on the master node - has_tensorboard = is_tensorboard_available() - if has_tensorboard and jax.process_index() == 0: - try: - from flax.metrics.tensorboard import SummaryWriter - - summary_writer = SummaryWriter(log_dir=Path(training_args.output_dir)) - except ImportError as ie: - has_tensorboard = False - logger.warning( - f"Unable to display metrics through TensorBoard because some package are not installed: {ie}" - ) - else: - logger.warning( - "Unable to display metrics through TensorBoard because the package is not installed: " - "Please run pip install tensorboard to enable." - ) - - # enable wandb tracking - has_wandb = find_spec("wandb") is not None - if jax.process_index() == 0 and has_wandb and ("wandb" in training_args.report_to): - try: - import wandb - wandb.init( - entity="wandb", - project="hf-flax-gpt-neo-copilot", - sync_tensorboard=True - ) - wandb.config.update(training_args) - wandb.config.update(model_args) - wandb.config.update(data_args) - except ImportError as e: - print(e) - has_wandb = False - - - # Initialize our training - rng = jax.random.PRNGKey(training_args.seed) - rng, dropout_rng = jax.random.split(rng) - - # Store some constant - num_epochs = int(training_args.num_train_epochs) - train_batch_size = int(training_args.per_device_train_batch_size) * jax.device_count() * training_args.gradient_accumulation_steps - eval_batch_size = int(training_args.per_device_eval_batch_size) * jax.device_count() - # steps_per_epoch = len(train_dataset) // train_batch_size - total_train_steps = training_args.max_steps - - # Create learning rate schedule - linear_decay_lr_schedule_fn = create_learning_rate_fn( - total_train_steps, - train_batch_size, - training_args.warmup_steps, - training_args.learning_rate, - ) - - # We use Optax's "masking" functionality to not apply weight decay - # to bias and LayerNorm scale parameters. decay_mask_fn returns a - # mask boolean with the same structure as the parameters. - # The mask is True for parameters that should be decayed. - # Note that this mask is specifically adapted for FlaxGPT2. - # For other models, one should correct the layer norm parameter naming - # accordingly. - def decay_mask_fn(params): - flat_params = traverse_util.flatten_dict(params) - flat_mask = { - path: (path[-1] != "bias" and path[-2:] not in [("ln_1", "scale"), ("ln_2", "scale"), ("ln_f", "scale")]) - for path in flat_params - } - return traverse_util.unflatten_dict(flat_mask) - - # create optimizer - if training_args.adafactor: - # We use the default parameters here to initialize adafactor, - # For more details about the parameters please check https://github.com/deepmind/optax/blob/ed02befef9bf81cbbf236be3d2b0e032e9ed4a40/optax/_src/alias.py#L74 - optimizer = optax.adafactor( - learning_rate=linear_decay_lr_schedule_fn, - ) - else: - optimizer = optax.adamw( - learning_rate=linear_decay_lr_schedule_fn, - b1=training_args.adam_beta1, - b2=training_args.adam_beta2, - eps=training_args.adam_epsilon, - weight_decay=training_args.weight_decay, - mask=decay_mask_fn, - ) - if training_args.gradient_accumulation_steps > 1: - optimizer = optax.MultiSteps(optimizer, training_args.gradient_accumulation_steps) - grad_accum_steps = training_args.gradient_accumulation_steps - - # Setup train state - state = TrainState.create(apply_fn=model.__call__, params=model.params, tx=optimizer, dropout_rng=dropout_rng) - - if training_args.resume_from_checkpoint: - state, resume_step = restore_checkpoint(training_args.resume_from_checkpoint, state) - else: - resume_step = 0 - - def loss_fn(logits, labels): - shift_logits = logits[..., :-1, :] - shift_labels = labels[..., 1:] - loss = optax.softmax_cross_entropy(shift_logits, onehot(shift_labels, shift_logits.shape[-1])) - return loss.mean() - - # Define gradient update step fn - def train_step(state, batch): - dropout_rng, new_dropout_rng = jax.random.split(state.dropout_rng) - - def compute_loss(params): - labels = batch.pop("labels") - logits = state.apply_fn(**batch, params=params, dropout_rng=dropout_rng, train=True)[0] - loss = loss_fn(logits, labels) - return loss - - grad_fn = jax.value_and_grad(compute_loss) - loss, grad = grad_fn(state.params) - grad = jax.lax.pmean(grad, "batch") - - new_state = state.apply_gradients(grads=grad, dropout_rng=new_dropout_rng) - - metrics = {"loss": loss, "learning_rate": linear_decay_lr_schedule_fn(state.step // grad_accum_steps)} - metrics = jax.lax.pmean(metrics, axis_name="batch") - - return new_state, metrics - - # Define eval fn - def eval_step(params, batch): - labels = batch.pop("labels") - logits = model(**batch, params=params, train=False)[0] - loss = loss_fn(logits, labels) - - # summarize metrics - metrics = {"loss": loss} - metrics = jax.lax.pmean(metrics, axis_name="batch") - return metrics - - # Create parallel version of the train and eval step - p_train_step = jax.pmap(train_step, "batch", donate_argnums=(0,)) - p_eval_step = jax.pmap(eval_step, "batch") - - # Replicate the train state on each device - state = state.replicate() - - logger.info("***** Running training *****") - # logger.info(f" Num examples = {len(train_dataset)}") - logger.info(f" Num Epochs = {num_epochs}") - logger.info(f" Instantaneous batch size per device = {training_args.per_device_train_batch_size}") - logger.info(f" Total train batch size (w. parallel, distributed and grad_accum) = {train_batch_size}") - logger.info(f" Total optimization steps = {total_train_steps}") - - if not training_args.skip_memory_metrics: - server = jax.profiler.start_server(9999) - - train_time = 0 - train_metrics = [] - # TODO: figure out training duration - steps = tqdm(range(total_train_steps*grad_accum_steps), desc=f"Step ... (1/{total_train_steps})", position=0, initial=resume_step) - for step in range(total_train_steps): - # ======================== Training ================================ - train_start = time.time() - rng, input_rng = jax.random.split(rng) - - cur_step = step - # skip to the step from which we are resuming - if cur_step < resume_step: - continue - - # samples = advance_iter_and_group_samples(iter(tokenized_dataset), int(training_args.per_device_train_batch_size) * jax.device_count(), block_size) - # batch = shard(make_batch(samples)) - batch = shard(next(train_dl)) - # logger.info(f"{batch['input_ids'].shape}") - state, train_metric = p_train_step(state, batch) - train_metrics.append(train_metric) - if step % grad_accum_steps == 0: - steps.update(1) - - if cur_step % (training_args.logging_steps * grad_accum_steps)== 0 and cur_step > 0: - # Save metrics - train_metric = unreplicate(train_metric) - train_time += time.time() - train_start - if has_tensorboard and jax.process_index() == 0: - write_train_metric(summary_writer, train_metrics, train_time, cur_step) - if has_wandb and jax.process_index() == 0 and ("wandb" in training_args.report_to): - # TODO: add accumulation of metrics - _metrics = {k if k=="learning_rate" else f"train_{k}":mb_item(v.mean()) for k, v in train_metric.items()} - wandb.log({"training_step":cur_step, **_metrics}, commit=True) - - steps.write( - f"Step... ({cur_step} | Loss: {train_metric['loss'].mean()}, Learning Rate: {train_metric['learning_rate'].mean()})" - ) - - train_metrics = [] - - if cur_step % (training_args.eval_steps * grad_accum_steps) == 0 and cur_step > 0 and training_args.do_eval: - # ======================== Evaluating ============================== - eval_metrics = [] - eval_loader = data_loader(input_rng, eval_dataset, eval_batch_size) - eval_steps = len(eval_dataset) // eval_batch_size - for _ in tqdm(range(eval_steps), desc="Evaluating...", position=2, leave=False): - # Model forward - batch = next(eval_loader) - metrics = p_eval_step(state.params, batch) - eval_metrics.append(metrics) - - # normalize eval metrics - eval_metrics = get_metrics(eval_metrics) - eval_metrics = jax.tree_map(jnp.mean, eval_metrics) - - try: - eval_metrics["perplexity"] = math.exp(eval_metrics["loss"]) - except OverflowError: - eval_metrics["perplexity"] = float("inf") - - # Print metrics and update progress bar - desc = f"Step... ({cur_step} | Eval Loss: {eval_metrics['loss']} | Eval Perplexity: {eval_metrics['perplexity']})" - steps.write(desc) - steps.desc = desc - - # Save metrics - if has_tensorboard and jax.process_index() == 0: - # cur_step = epoch * (len(train_dataset) // train_batch_size) - write_eval_metric(summary_writer, eval_metrics, cur_step) - if has_wandb and jax.process_index() == 0 and ("wandb" in training_args.report_to): - _metrics = {f"eval_{k}":mb_item(v) for k, v in eval_metrics.items()} - wandb.log({"eval_step":cur_step, **_metrics}) - - if cur_step % (training_args.save_steps * grad_accum_steps) == 0 and cur_step > 0: - # save checkpoint after each epoch and push checkpoint to the hub - if jax.process_index() == 0: - save_checkpoint(model, training_args.output_dir, state, push_to_hub=training_args.push_to_hub) - if training_args.save_total_limit is not None: - rotate_checkpoints(training_args.output_dir, training_args.save_total_limit) - - # save model after training is over - save_checkpoint(model, training_args.output_dir, state, with_opt=False, push_to_hub=training_args.push_to_hub) - - - - -if __name__ == "__main__": - main() - diff --git a/run_clm_streaming_wikitext.sh b/run_clm_streaming_wikitext.sh deleted file mode 100644 index be0aaba..0000000 --- a/run_clm_streaming_wikitext.sh +++ /dev/null @@ -1,37 +0,0 @@ -#! /bin/bash -./run_clm_streaming_flax_clean.py \ - --output_dir $HOME/gpt-neo-125M-test \ - --model_name_or_path="EleutherAI/gpt-neo-125M" \ - --dataset_name="wikitext" \ - --dataset_config_name="wikitext-103-raw-v1" \ - --text_column_name="text" \ - --do_train --do_eval \ - --block_size="128" \ - --per_device_train_batch_size="8" \ - --per_device_eval_batch_size="8" \ - --preprocessing_num_workers="8" \ - --learning_rate="6e-4" \ - --max_steps 500 \ - --warmup_steps 150 \ - --decay_steps 250 \ - --adam_beta1="0.9" \ - --adam_beta2="0.95" \ - --weight_decay="0.01" \ - --overwrite_output_dir \ - --logging_steps="10" \ - --eval_steps="50" \ - --push_to_hub="False" \ - --report_to="all" \ - --dtype="bfloat16" \ - --skip_memory_metrics="False" \ - --save_steps="50" \ - --save_total_limit 2 \ - --gradient_accumulation_steps 8 \ - --report_to="wandb" \ - --run_name="testing-mini" \ - --max_eval_samples 100 \ - --save_optimizer true \ - # --resume_from_checkpoint $HOME/gpt-neo-125M-test/ckpt-800 \ - # --adafactor \ - # --max_train_samples="10000" \ - diff --git a/run_clm_wikitext.sh b/run_clm_wikitext.sh deleted file mode 100644 index e73253d..0000000 --- a/run_clm_wikitext.sh +++ /dev/null @@ -1,35 +0,0 @@ -#! /bin/bash -./run_clm_flax.py \ - --output_dir $HOME/tmp/gpt-neo-125M-test-2 \ - --model_name_or_path="EleutherAI/gpt-neo-125M" \ - --dataset_name="wikitext" \ - --dataset_config_name="wikitext-2-raw-v1" \ - --text_column_name="text" \ - --do_train --do_eval \ - --block_size="128" \ - --per_device_train_batch_size="8" \ - --per_device_eval_batch_size="16" \ - --preprocessing_num_workers="8" \ - --learning_rate="2e-5" \ - --adafactor \ - --warmup_steps="100" \ - --adam_beta1="0.9" \ - --adam_beta2="0.98" \ - --weight_decay="0.01" \ - --overwrite_output_dir \ - --num_train_epochs="10" \ - --logging_steps="10" \ - --eval_steps="10" \ - --push_to_hub="False" \ - --report_to="none" \ - --run_name="test-non-streaming" \ - --dtype="bfloat16" \ - --skip_memory_metrics="False" \ - --save_steps="200" \ - --save_strategy epoch \ - --save_total_limit 2 \ - --gradient_accumulation_steps 8 \ - --save_optimizer true \ - --resume_from_checkpoint $HOME/tmp/gpt-neo-125M-test-2/ckpt-2591 \ - # --max_train_samples="10000" \ - # --max_eval_samples="1000" diff --git a/scripts/notify_repo_owners.py b/scripts/notify_repo_owners.py deleted file mode 100644 index 5aa9dae..0000000 --- a/scripts/notify_repo_owners.py +++ /dev/null @@ -1,45 +0,0 @@ -import os - -import pandas as pd - -from fastcore.script import * -from ghapi.all import GhApi - -GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN") -ISSUE_TITLE = "Participation in an Open Source Language Modeling Dataset" -ISSUE_BODY = """Hi there, your repository has been selected to be included in an effort -to train an open source version of GitHub and OpenAI's [Copilot tool](https://copilot.github.com/). -You can find more information on our project [here](https://github.com/ncoop57/gpt-code-clippy). - -If you are the owner/admin of this repository and would like to opt-out of this, -please downvote this issue before July 9th and we will remove your repository -from our list. We will comment an acknowledgement in this issue if you choose -to opt-out on July 9th. If you have any questions, please open an issue on our -[repository](https://github.com/ncoop57/gpt-code-clippy/issues/new). -""" - -REPOS = [ - "ProgrammingSpace/test-repo-3", - "ncoop57/deep_parking", - "ncoop57/test-repo", - "ncoop57/recipe_name_suggester", - "ncoop57/test-repo-2", -] - -# Open issue on repo using custom title and body -def open_issue(owner, repo): - api = GhApi(owner=owner, repo=repo, token=GITHUB_TOKEN) - api.issues.create(title=ISSUE_TITLE, body=ISSUE_BODY) - - -@call_parse -def main(repos_path: Param("Path to the csv containing all of the repos", str)): - """ - Use pandas dataframe from the repos path to open issues in each of them. - """ - df = pd.read_csv(repos_path) - - # Loop through repos and open issue for each repo - for _, row in df.iterrows(): - owner, repo = row["name"].split("/") - open_issue(owner=owner, repo=repo) diff --git a/scripts/read_repo_replies.py b/scripts/read_repo_replies.py deleted file mode 100644 index 5c2fa60..0000000 --- a/scripts/read_repo_replies.py +++ /dev/null @@ -1,27 +0,0 @@ -import os - -from ghapi.all import GhApi - -GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN") -ISSUE_TITLE = "Participation in an Open Source Language Modeling Dataset" -REPOS = [ - "ProgrammingSpace/test-repo-3", - "ncoop57/deep_parking", - "ncoop57/test-repo", - "ncoop57/recipe_name_suggester", - "ncoop57/test-repo-2", -] - -for r in REPOS: - print(r) - owner, repo = r.split("/") - api = GhApi(owner=owner, repo=repo, token=GITHUB_TOKEN) - issues = api.issues.list_for_repo(owner=owner, repo=repo, state="all") - for i in issues: - if i.title == ISSUE_TITLE: - if i.reactions["-1"] > 0: - print(f"{r} is opting out") - api.issues.create_comment( - i.number, body="Thank you, your repository has been removed." - ) - api.issues.update(i.number, state="closed") diff --git a/scripts/add_new_tokens.py b/training/add_new_tokens.py similarity index 100% rename from scripts/add_new_tokens.py rename to training/add_new_tokens.py diff --git a/code_clippy.py b/training/code_clippy.py similarity index 100% rename from code_clippy.py rename to training/code_clippy.py diff --git a/scripts/new_tokens.json b/training/new_tokens.json similarity index 100% rename from scripts/new_tokens.json rename to training/new_tokens.json diff --git a/run_clm_flax.py b/training/run_clm_flax.py similarity index 100% rename from run_clm_flax.py rename to training/run_clm_flax.py diff --git a/run_clm_gpt_neo_13b.sh b/training/run_clm_gpt_neo_13b.sh similarity index 100% rename from run_clm_gpt_neo_13b.sh rename to training/run_clm_gpt_neo_13b.sh diff --git a/run_clm_gpt_neo_27b.sh b/training/run_clm_gpt_neo_27b.sh similarity index 100% rename from run_clm_gpt_neo_27b.sh rename to training/run_clm_gpt_neo_27b.sh diff --git a/run_clm_streaming_125m_1e-4lr_1024bs.sh b/training/run_clm_streaming_125m_1e-4lr_1024bs.sh similarity index 100% rename from run_clm_streaming_125m_1e-4lr_1024bs.sh rename to training/run_clm_streaming_125m_1e-4lr_1024bs.sh diff --git a/run_clm_streaming_1_3b_1e-4lr_1024bs.sh b/training/run_clm_streaming_1_3b_1e-4lr_1024bs.sh similarity index 100% rename from run_clm_streaming_1_3b_1e-4lr_1024bs.sh rename to training/run_clm_streaming_1_3b_1e-4lr_1024bs.sh diff --git a/run_clm_streaming_flax_clean.py b/training/run_clm_streaming_flax_clean.py similarity index 100% rename from run_clm_streaming_flax_clean.py rename to training/run_clm_streaming_flax_clean.py diff --git a/run_clm_streaming_flax_v2.py b/training/run_clm_streaming_flax_v2.py similarity index 100% rename from run_clm_streaming_flax_v2.py rename to training/run_clm_streaming_flax_v2.py diff --git a/utils.py b/training/utils.py similarity index 100% rename from utils.py rename to training/utils.py From 51dc4c38529de143e161539fbd024a80bc0696da Mon Sep 17 00:00:00 2001 From: ncoop57 Date: Wed, 14 Jul 2021 20:22:13 -0400 Subject: [PATCH 11/15] Update readme with outline --- README.md | 44 ++++++++------------------------------------ 1 file changed, 8 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 37db34f..837bebe 100644 --- a/README.md +++ b/README.md @@ -1,47 +1,19 @@ # GPT-Code-Clippy (GPT-CC) -# Open Source GitHub Copilot for auto generating code +## Open Source GitHub Copilot for auto generating code I would like to train an open source version of the new awesome GitHub Copilot AI tool, which is based on GPT3. Similar to the awesome people behind GPT-Neo, having such an open source model would greatly help researchers understand what this type of biases and limitations this kind of code autocompletion model might have such as generating insecure code (i do research in this area and i know my team would love an open sourced version to run experiments on, i.e. try and break it 🤓) -## 2. Language +## Getting the data -The model will be trained on different programming languages such as C, C++, java, python, etc. +### Downloading the data -## 3. Model +### Further processing the data -GPT-Neo +## Finetuning the model -## 4. Datasets +## Evaluating the model -Datasets that contain hopefully high quality source code +## Using the model -Possible links to publicly available datasets include: -- https://huggingface.co/datasets/code_search_net -- https://huggingface.co/datasets?search=code_x - -Some additional datasets may need creating that are not just method level. - -## 5. Training scripts - -I believe the standard CLM language model script would do for this. - -We can make use of https://www.github.com/huggingface/transformers/tree/master/examples%2Fflax%2Flanguage-modeling%2Frun_clm_flax.py - -## 6. (Optional) Challenges - -The data additional data may be a challenge. From what I can see in copilot, it looks to be training on entire files, not code snippets. There are file level datasets that exist but they are a few years old and i don't think they cover many programming languages. The ones I listed above have multiple languages but are only methods. - -However, githubs API is pretty easy to use and so it would be pretty easy to create one from scratch, especially if we get some insights into how the copilot dataset was generated 🤓 - -## 7. (Optional) Desired project outcome - -I'd love to have this open source model setup in a similar Visual Studio Code extension to the GitHub Copilot one. I've actually made a tutorial on doing this using the GPT-Neo model, so we could easily clean it up and release it free of charge forever because from what I've seen on Twitter the GitHub Copilot might eventually be put behind a paywall 😢. - -## 8. (Optional) Reads - -The following links can be useful to better understand the project and -what has previously been done. - -- https://github.blog/2021-06-29-introducing-github-copilot-ai-pair-programmer/ -- https://youtu.be/nC3NrhoNeP4 (tutorial on how we could setup the demo of the model once it's done cooking) +## Citations \ No newline at end of file From 9c5c7aa4e143063907c0153497390211d8e9eb8f Mon Sep 17 00:00:00 2001 From: ncoop57 Date: Wed, 14 Jul 2021 20:36:03 -0400 Subject: [PATCH 12/15] Even more reorganizing --- .../evaluation => evaluation}/evaluate.py | 40 ++++++++++--------- .../apps_utils/generate_gpt_codes.py | 0 .../evaluation/apps_utils/reindent.py | 0 .../apps_utils/test_one_solution.py | 0 .../evaluation/apps_utils/testing_util.py | 0 .../evaluation/code_search_net.py | 0 .../evaluation/concode.py | 0 .../evaluation/human_eval.jsonl | 0 .../evaluation/human_eval.jsonl_results.jsonl | 0 .../evaluation/human_eval_bench.py | 0 .../evaluation/metrics/bleu.py | 0 .../evaluation/metrics/extrinsic_eval.py | 0 12 files changed, 22 insertions(+), 18 deletions(-) rename {data_processing/evaluation => evaluation}/evaluate.py (87%) rename {data_processing => evaluation}/evaluation/apps_utils/generate_gpt_codes.py (100%) rename {data_processing => evaluation}/evaluation/apps_utils/reindent.py (100%) rename {data_processing => evaluation}/evaluation/apps_utils/test_one_solution.py (100%) rename {data_processing => evaluation}/evaluation/apps_utils/testing_util.py (100%) rename {data_processing => evaluation}/evaluation/code_search_net.py (100%) rename {data_processing => evaluation}/evaluation/concode.py (100%) rename {data_processing => evaluation}/evaluation/human_eval.jsonl (100%) rename {data_processing => evaluation}/evaluation/human_eval.jsonl_results.jsonl (100%) rename {data_processing => evaluation}/evaluation/human_eval_bench.py (100%) rename {data_processing => evaluation}/evaluation/metrics/bleu.py (100%) rename {data_processing => evaluation}/evaluation/metrics/extrinsic_eval.py (100%) diff --git a/data_processing/evaluation/evaluate.py b/evaluation/evaluate.py similarity index 87% rename from data_processing/evaluation/evaluate.py rename to evaluation/evaluate.py index 4fbd421..c2e5cee 100644 --- a/data_processing/evaluation/evaluate.py +++ b/evaluation/evaluate.py @@ -12,30 +12,34 @@ from human_eval.data import write_jsonl, read_problems from pathlib import Path from metrics.extrinsic_eval import compute_metrics from subprocess import check_output -from transformers import AutoTokenizer, AutoModelWithLMHead +from transformers import ( + AutoTokenizer, + FlaxGPTNeoForCausalLM, +) bleu = load_metric("sacrebleu") -tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neo-125M") -model = AutoModelWithLMHead.from_pretrained( - "/home/nathan/gpt-code-clippy/data/APPS/models/1.5B" +# tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neo-125M") +# model = AutoModelWithLMHead.from_pretrained( +# "/home/nathan/gpt-code-clippy/data/APPS/models/1.5B" +# ) + +MAX_TOKENs = 1_024 +model_name_or_path = "EleutherAI/gpt-neo-125M" + +tokenizer = AutoTokenizer.from_pretrained( + model_name_or_path, padding_side="left", pad_token="<|endoftext|>" ) +model = FlaxGPTNeoForCausalLM.from_pretrained( + model_name_or_path, + pad_token_id=50256, +).to("cuda") def generate_text(prompt): - # print(prompt) - input_ids = torch.LongTensor(tokenizer.encode(prompt, verbose=False)).unsqueeze( - 0 - ) # .cuda() - output_ids = model.generate( - input_ids, - num_beams=2, - early_stopping=True, - max_length=1024 - len(input_ids), - ) - output_str = tokenizer.decode(output_ids[0]) - return output_str - # # "a", "=", "b", "\n", "y", "=", "a", "+", "1" - # return "a = b \n y = a + 1" + inputs = tokenizer(prompt, return_tensors="jax").to("cuda") + output_seq = model.generate(input_ids=inputs.input_ids, max_length=1_024) + + return tokenizer.decode(output_seq["sequences"][0]) def _eval_concode(path): diff --git a/data_processing/evaluation/apps_utils/generate_gpt_codes.py b/evaluation/evaluation/apps_utils/generate_gpt_codes.py similarity index 100% rename from data_processing/evaluation/apps_utils/generate_gpt_codes.py rename to evaluation/evaluation/apps_utils/generate_gpt_codes.py diff --git a/data_processing/evaluation/apps_utils/reindent.py b/evaluation/evaluation/apps_utils/reindent.py similarity index 100% rename from data_processing/evaluation/apps_utils/reindent.py rename to evaluation/evaluation/apps_utils/reindent.py diff --git a/data_processing/evaluation/apps_utils/test_one_solution.py b/evaluation/evaluation/apps_utils/test_one_solution.py similarity index 100% rename from data_processing/evaluation/apps_utils/test_one_solution.py rename to evaluation/evaluation/apps_utils/test_one_solution.py diff --git a/data_processing/evaluation/apps_utils/testing_util.py b/evaluation/evaluation/apps_utils/testing_util.py similarity index 100% rename from data_processing/evaluation/apps_utils/testing_util.py rename to evaluation/evaluation/apps_utils/testing_util.py diff --git a/data_processing/evaluation/code_search_net.py b/evaluation/evaluation/code_search_net.py similarity index 100% rename from data_processing/evaluation/code_search_net.py rename to evaluation/evaluation/code_search_net.py diff --git a/data_processing/evaluation/concode.py b/evaluation/evaluation/concode.py similarity index 100% rename from data_processing/evaluation/concode.py rename to evaluation/evaluation/concode.py diff --git a/data_processing/evaluation/human_eval.jsonl b/evaluation/evaluation/human_eval.jsonl similarity index 100% rename from data_processing/evaluation/human_eval.jsonl rename to evaluation/evaluation/human_eval.jsonl diff --git a/data_processing/evaluation/human_eval.jsonl_results.jsonl b/evaluation/evaluation/human_eval.jsonl_results.jsonl similarity index 100% rename from data_processing/evaluation/human_eval.jsonl_results.jsonl rename to evaluation/evaluation/human_eval.jsonl_results.jsonl diff --git a/data_processing/evaluation/human_eval_bench.py b/evaluation/evaluation/human_eval_bench.py similarity index 100% rename from data_processing/evaluation/human_eval_bench.py rename to evaluation/evaluation/human_eval_bench.py diff --git a/data_processing/evaluation/metrics/bleu.py b/evaluation/evaluation/metrics/bleu.py similarity index 100% rename from data_processing/evaluation/metrics/bleu.py rename to evaluation/evaluation/metrics/bleu.py diff --git a/data_processing/evaluation/metrics/extrinsic_eval.py b/evaluation/evaluation/metrics/extrinsic_eval.py similarity index 100% rename from data_processing/evaluation/metrics/extrinsic_eval.py rename to evaluation/evaluation/metrics/extrinsic_eval.py From c5966efd865a153fc478e882d5c22febf3101668 Mon Sep 17 00:00:00 2001 From: Mrinal Mathur Date: Thu, 15 Jul 2021 15:12:58 +0530 Subject: [PATCH 13/15] Adding documentation on usage --- README.md | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 837bebe..89bc329 100644 --- a/README.md +++ b/README.md @@ -16,4 +16,62 @@ I would like to train an open source version of the new awesome GitHub Copilot A ## Using the model -## Citations \ No newline at end of file +Possible links to publicly available datasets include: +- https://huggingface.co/datasets/code_search_net +- https://huggingface.co/datasets?search=code_x + +Some additional datasets may need creating that are not just method level. + +## 5. Training scripts + +I believe the standard CLM language model script would do for this. + +We can make use of https://www.github.com/huggingface/transformers/tree/master/examples%2Fflax%2Flanguage-modeling%2Frun_clm_flax.py + +for training the scripts you can run: +`python streaming_flax.py ` + + +## 6. Usage + + +code for running the code generation is done by using: +`bash run_cln_straming.sh` + +run_cln_straming.sh contains all the hyperparameters and will be used to generate code. + +we have also generated the code for the following languages: +python +javascript +c++ +c +java + +We have used GPT-Neo using 13B and 27B parameter settings. + +to run for following files: + +13B: +`bash run_cln_gpt_neo_13b.sh` + +27B: +`bash run_cln_gpt_neo_27b.sh` + + +## 7. (Optional) Challenges + +The data additional data may be a challenge. From what I can see in copilot, it looks to be training on entire files, not code snippets. There are file level datasets that exist but they are a few years old and i don't think they cover many programming languages. The ones I listed above have multiple languages but are only methods. + +However, githubs API is pretty easy to use and so it would be pretty easy to create one from scratch, especially if we get some insights into how the copilot dataset was generated 🤓 + +## 8. (Optional) Desired project outcome + +I'd love to have this open source model setup in a similar Visual Studio Code extension to the GitHub Copilot one. I've actually made a tutorial on doing this using the GPT-Neo model, so we could easily clean it up and release it free of charge forever because from what I've seen on Twitter the GitHub Copilot might eventually be put behind a paywall 😢. + +## 9. (Optional) Reads + +The following links can be useful to better understand the project and +what has previously been done. + +- https://github.blog/2021-06-29-introducing-github-copilot-ai-pair-programmer/ +- https://youtu.be/nC3NrhoNeP4 (tutorial on how we could setup the demo of the model once it's done cooking) From 0d63ea7fa9d43766876a6633f790906715dbfe6f Mon Sep 17 00:00:00 2001 From: Mrinal Mathur Date: Thu, 15 Jul 2021 17:46:38 +0530 Subject: [PATCH 14/15] Updated training file information --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 89bc329..8c1bdca 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ I believe the standard CLM language model script would do for this. We can make use of https://www.github.com/huggingface/transformers/tree/master/examples%2Fflax%2Flanguage-modeling%2Frun_clm_flax.py for training the scripts you can run: -`python streaming_flax.py ` +`python run_clm_streaming_flax_v2.py ` ## 6. Usage From a2e6c08a3cfbd574172b93da35670d7935a1d436 Mon Sep 17 00:00:00 2001 From: ncoop57 Date: Thu, 15 Jul 2021 15:59:42 -0400 Subject: [PATCH 15/15] WIP on human eval evaluation --- evaluation/evaluate.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/evaluation/evaluate.py b/evaluation/evaluate.py index c2e5cee..788f585 100644 --- a/evaluation/evaluate.py +++ b/evaluation/evaluate.py @@ -4,13 +4,14 @@ import pandas as pd # import apps.eval.reident -from apps_utils.generate_gpt_codes import generate_prompt -from apps_utils.test_one_solution import eval_and_save_problems +# from apps_utils.generate_gpt_codes import generate_prompt +# from apps_utils.test_one_solution import eval_and_save_problems from datasets import load_dataset, load_metric from fastcore.script import * -from human_eval.data import write_jsonl, read_problems +from human_eval.data import HUMAN_EVAL, write_jsonl, read_problems +from human_eval.evaluation import evaluate_functional_correctness from pathlib import Path -from metrics.extrinsic_eval import compute_metrics +# from metrics.extrinsic_eval import compute_metrics from subprocess import check_output from transformers import ( AutoTokenizer, @@ -32,11 +33,11 @@ tokenizer = AutoTokenizer.from_pretrained( model = FlaxGPTNeoForCausalLM.from_pretrained( model_name_or_path, pad_token_id=50256, -).to("cuda") +) def generate_text(prompt): - inputs = tokenizer(prompt, return_tensors="jax").to("cuda") + inputs = tokenizer(prompt, return_tensors="jax")#.to("cuda") output_seq = model.generate(input_ids=inputs.input_ids, max_length=1_024) return tokenizer.decode(output_seq["sequences"][0]) @@ -113,7 +114,8 @@ def _eval_apps(path): def _eval_human_eval(path): - problems = read_problems() + + problems = read_problems(str(path)) num_samples_per_task = 1 samples = [ dict( @@ -125,13 +127,14 @@ def _eval_human_eval(path): ] write_jsonl("human_eval.jsonl", samples) # execute bash command to run eval script - results = check_output( - [ - "python", - path / "evaluate_functional_correctness.py", - "human_eval.jsonl", - ] - ).decode("utf-8") + results = evaluate_functional_correctness("human_eval.jsonl", [1], 4, 3.0, str(path)) + # results = check_output( + # [ + # "python", + # path / "evaluate_functional_correctness.py", + # "human_eval.jsonl", + # ] + # ).decode("utf-8") print(results) @@ -146,8 +149,8 @@ def main( apps_path = Path(apps_path) human_eval_path = Path(human_eval_path) # _eval_concode(concode_path) - # _eval_human_eval(human_eval_path) - _eval_apps(apps_path) + _eval_human_eval(human_eval_path) + # _eval_apps(apps_path) # dataset = load_dataset("json", data_files=str(concode_path / "test.json")) # print(dataset) # results = bleu.compute(predictions=predictions, references=references)