mirror of
https://github.com/CodedotAl/gpt-code-clippy.git
synced 2024-08-17 02:40:33 +03:00
Add filter dataset version
This commit is contained in:
parent
9dc465526a
commit
e5058af709
202
data_processing/code_clippy_filter.py
Normal file
202
data_processing/code_clippy_filter.py
Normal file
@ -0,0 +1,202 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2020 The HuggingFace Datasets Authors and the CodeClippy team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
CodeClippy dataset - opensource code from Github. Scrapped July 7 2021.
|
||||
More to add here.
|
||||
"""
|
||||
|
||||
import os
|
||||
import io
|
||||
from typing import List
|
||||
import jsonlines
|
||||
import json
|
||||
import zstandard as zstd
|
||||
from pathlib import Path
|
||||
|
||||
import datasets
|
||||
|
||||
|
||||
# TODO: Add BibTeX citation
|
||||
# Find for instance the citation on arxiv or on the dataset repo/website
|
||||
_CITATION = """\
|
||||
@InProceedings{huggingface:dataset,
|
||||
title = {Code Clippy},
|
||||
author={CodeClippy team and all the opensource devs around the globe
|
||||
},
|
||||
year={2021}
|
||||
}
|
||||
"""
|
||||
|
||||
_DESCRIPTION = """
|
||||
CodeClippy dataset - opensource code from Github. Scrapped July 7 2021.
|
||||
"""
|
||||
|
||||
# TODO: Add a link to an official homepage for the dataset here
|
||||
_HOMEPAGE = ""
|
||||
|
||||
# TODO: Add the licence for the dataset here if you can find it
|
||||
_LICENSE = ""
|
||||
|
||||
# TODO: Add link to the official dataset URLs here (once we have those)
|
||||
# The HuggingFace dataset library don't host the datasets but only point to the original files
|
||||
# This can be an arbitrary nested dict/list of URLs (see below in `_split_generators` method)
|
||||
_URLs = {
|
||||
"https://huggingface.co/great-new-dataset-first_domain.zip",
|
||||
}
|
||||
|
||||
|
||||
class CodeClippy(datasets.GeneratorBasedBuilder):
|
||||
"""CodeClippy dataset - opensource code from Github. Scrapped July 7 2021."""
|
||||
|
||||
VERSION = datasets.Version("0.1.0")
|
||||
|
||||
# This is an example of a dataset with multiple configurations.
|
||||
# If you don't want/need to define several sub-sets in your dataset,
|
||||
# just remove the BUILDER_CONFIG_CLASS and the BUILDER_CONFIGS attributes.
|
||||
|
||||
# If you need to make complex sub-parts in the datasets with configurable options
|
||||
# You can create your own builder configuration class to store attribute, inheriting from datasets.BuilderConfig
|
||||
# BUILDER_CONFIG_CLASS = MyBuilderConfig
|
||||
|
||||
# You will be able to load one or the other configurations in the following list with
|
||||
# data = datasets.load_dataset('my_dataset', 'first_domain')
|
||||
# data = datasets.load_dataset('my_dataset', 'second_domain')
|
||||
# BUILDER_CONFIGS = [
|
||||
# datasets.BuilderConfig(name="first_domain", version=VERSION, description="This part of my dataset covers a first domain"),
|
||||
# datasets.BuilderConfig(name="second_domain", version=VERSION, description="This part of my dataset covers a second domain"),
|
||||
# ]
|
||||
|
||||
# DEFAULT_CONFIG_NAME = "first_domain"
|
||||
|
||||
def _info(self):
|
||||
features = datasets.Features(
|
||||
{
|
||||
"id": datasets.Value("int64"),
|
||||
"text": datasets.Value("string"),
|
||||
"repo_name": datasets.Value("string"),
|
||||
"stars": datasets.Value("string"),
|
||||
"repo_language": datasets.Value("string"),
|
||||
"file_name": datasets.Value("string"),
|
||||
"mime_type": datasets.Value("string"),
|
||||
}
|
||||
)
|
||||
return datasets.DatasetInfo(
|
||||
description=_DESCRIPTION,
|
||||
features=features,
|
||||
homepage=_HOMEPAGE,
|
||||
license=_LICENSE,
|
||||
)
|
||||
|
||||
def _split_generators(self, dl_manager):
|
||||
"""Returns SplitGenerators."""
|
||||
# dl_manager is a datasets.download.DownloadManager that can be used to download and extract URLs
|
||||
# It can accept any type or nested list/dict and will give back the same structure with the url replaced with path to local files.
|
||||
# By default the archives will be extracted and a path to a cached folder where they are extracted is returned instead of the archive
|
||||
|
||||
# data_dir = dl_manager.download_and_extract(_URLs)
|
||||
# filepath = dl_manager.download("https://gist.githubusercontent.com/ppisarczyk/43962d06686722d26d176fad46879d41/raw/211547723b4621a622fc56978d74aa416cbd1729/Programming_Languages_Extensions.json")
|
||||
# with open(filepath, "r") as f:
|
||||
# data = json.load(f)
|
||||
|
||||
# lang_exts = []
|
||||
# for i in data:
|
||||
# if "extensions" not in i:
|
||||
# continue
|
||||
# lang_exts.extend(i["extensions"])
|
||||
# self.lang_exts = set(lang_exts)
|
||||
self.lang_exts = {
|
||||
".lisp",
|
||||
".lsp",
|
||||
".f",
|
||||
".fs",
|
||||
".sh",
|
||||
".groovy",
|
||||
".r",
|
||||
".pl",
|
||||
".html",
|
||||
".css",
|
||||
".sql",
|
||||
".py",
|
||||
".c",
|
||||
".cpp",
|
||||
".h",
|
||||
".hpp",
|
||||
".jl",
|
||||
".java",
|
||||
".js",
|
||||
".ts",
|
||||
".cs",
|
||||
".go",
|
||||
".rs",
|
||||
".swift",
|
||||
".php",
|
||||
".dart",
|
||||
".kt",
|
||||
".m",
|
||||
".hs",
|
||||
".scala",
|
||||
".sc",
|
||||
".lua",
|
||||
".rb",
|
||||
}
|
||||
data_dir = self.config.data_dir
|
||||
return [
|
||||
datasets.SplitGenerator(
|
||||
name=datasets.Split.TRAIN,
|
||||
gen_kwargs={
|
||||
"filepaths": sorted(
|
||||
[
|
||||
str(fp)
|
||||
for fp in Path(f"{data_dir}/train").glob("*.jsonl.zst")
|
||||
]
|
||||
)
|
||||
},
|
||||
),
|
||||
datasets.SplitGenerator(
|
||||
name=datasets.Split.TEST,
|
||||
gen_kwargs={
|
||||
"filepaths": sorted(
|
||||
[str(fp) for fp in Path(f"{data_dir}/test").glob("*.jsonl.zst")]
|
||||
)
|
||||
},
|
||||
),
|
||||
datasets.SplitGenerator(
|
||||
name=datasets.Split.VALIDATION,
|
||||
gen_kwargs={
|
||||
"filepaths": sorted(
|
||||
[
|
||||
str(fp)
|
||||
for fp in Path(f"{data_dir}/validation").glob("*.jsonl.zst")
|
||||
]
|
||||
)
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
def _generate_examples(self, filepaths: List):
|
||||
"""Yields examples as (key, example) tuples."""
|
||||
id_ = 0
|
||||
dctx = zstd.ZstdDecompressor()
|
||||
for filepath in filepaths:
|
||||
with open(filepath, "rb") as f:
|
||||
f = dctx.stream_reader(f)
|
||||
f = io.TextIOWrapper(f, encoding="utf-8")
|
||||
f = jsonlines.Reader(f)
|
||||
for line in f:
|
||||
filename = line["meta"]["file_name"]
|
||||
start = filename.rfind(".")
|
||||
if filename[start:] in self.lang_exts:
|
||||
yield id_, {"id": id_, "text": line["text"], **line["meta"]}
|
||||
id_ += 1
|
Loading…
Reference in New Issue
Block a user