copy deploy to test on new framework version
This commit is contained in:
commit
548b20d7c8
10 changed files with 7809 additions and 0 deletions
2
.gitignore
vendored
Normal file
2
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
deploy
|
||||
README.md
|
||||
14
app/api-pipelines.yaml
Normal file
14
app/api-pipelines.yaml
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
apiVersion: "unified-platform.cs.hse.ru/v1"
|
||||
kind: APIComponent
|
||||
metadata:
|
||||
name: api-pipelines
|
||||
namespace: pu-mgalynchik-pa-mm25-copy
|
||||
spec:
|
||||
published: true
|
||||
pipelines:
|
||||
enabled: true
|
||||
restfulApi:
|
||||
auth:
|
||||
basic:
|
||||
credentials: synth-apis-cred
|
||||
identityPassThrough: true
|
||||
16
app/box.yaml
Normal file
16
app/box.yaml
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
apiVersion: "unified-platform.cs.hse.ru/v1"
|
||||
kind: DataBox
|
||||
metadata:
|
||||
name: synthdata-box
|
||||
namespace: pu-mgalynchik-pa-mm25-copy
|
||||
spec:
|
||||
s3DefaultStorage: {}
|
||||
---
|
||||
|
||||
apiVersion: "unified-platform.cs.hse.ru/v1"
|
||||
kind: DataBox
|
||||
metadata:
|
||||
name: users
|
||||
namespace: pu-mgalynchik-pa-mm25-copy
|
||||
spec:
|
||||
s3DefaultStorage: {}
|
||||
14
app/files-api.yaml
Normal file
14
app/files-api.yaml
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
apiVersion: "unified-platform.cs.hse.ru/v1"
|
||||
kind: APIComponent
|
||||
metadata:
|
||||
name: files-api
|
||||
namespace: pu-mgalynchik-pa-mm25-copy
|
||||
spec:
|
||||
published: true
|
||||
files:
|
||||
enabled: true
|
||||
restfulApi:
|
||||
auth:
|
||||
basic:
|
||||
credentials: synth-apis-cred
|
||||
identityPassThrough: true
|
||||
29
app/syntdata-test-api.yaml
Normal file
29
app/syntdata-test-api.yaml
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
|
||||
apiVersion: "unified-platform.cs.hse.ru/v1"
|
||||
kind: APIComponent
|
||||
metadata:
|
||||
name: train-generate-api
|
||||
namespace: pu-mgalynchik-pa-mm25-copy
|
||||
spec:
|
||||
published: true
|
||||
experimentPipeline:
|
||||
name: train-generate
|
||||
restfulApi:
|
||||
auth:
|
||||
basic:
|
||||
credentials: synth-apis-cred
|
||||
identityPassThrough: true
|
||||
apiSpec:
|
||||
inputs:
|
||||
- name: input_data
|
||||
description: "Путь до папки с датасетом для обучения"
|
||||
type:
|
||||
datatypes: [ "FILE" ]
|
||||
contentTypes: [ "text/csv" ]
|
||||
outputs:
|
||||
- name: report_file
|
||||
description: "Путь до папки куда пайплайн сохранит отчет об обучении"
|
||||
- name: output_data
|
||||
description: "Путь до папки куда пайплайн сохранит сгенерированные данные"
|
||||
- name: model
|
||||
description: "Путь до папки куда пайплайн сохранит модель"
|
||||
35
app/syntdata-test-pipe.yaml
Normal file
35
app/syntdata-test-pipe.yaml
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
apiVersion: "unified-platform.cs.hse.ru/v1"
|
||||
kind: ExperimentPipeline
|
||||
metadata:
|
||||
name: train-generate
|
||||
namespace: pu-mgalynchik-pa-mm25-copy
|
||||
spec:
|
||||
vars:
|
||||
- name: model
|
||||
- name: input_data
|
||||
- name: report_file
|
||||
- name: output_data
|
||||
stages:
|
||||
- name: synth-data-stage
|
||||
image:
|
||||
existingImageName: registry.platform-dev.stratpro.hse.ru/mm25-synthdata/synt_data_stage:c157543
|
||||
inputs:
|
||||
- name: input_data
|
||||
path: /unip/data/synthesize_data/input
|
||||
outputs:
|
||||
- name: report_file
|
||||
path: /unip/results/synthesize_data/report
|
||||
- name: output_data
|
||||
path: /unip/results/synthesize_data/output
|
||||
- name: model
|
||||
path: /unip/results/synthesize_data/model
|
||||
resourceLimits:
|
||||
cpu: "800m"
|
||||
memory: "4G"
|
||||
connectedBoxes:
|
||||
- name: synthdata-box
|
||||
path: /unip
|
||||
default: true
|
||||
mountS3Box:
|
||||
s3BoxName: synthdata-box
|
||||
|
||||
7044
data2/telecom/telecom_data.csv
Normal file
7044
data2/telecom/telecom_data.csv
Normal file
File diff suppressed because it is too large
Load diff
469
mm25-module-card.ipynb
Normal file
469
mm25-module-card.ipynb
Normal file
|
|
@ -0,0 +1,469 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Проверка работы с модулем 25 через API"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### Обучение модели и генерация данных"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"##### Файловый API"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import requests\n",
|
||||
"import json\n",
|
||||
"from requests.auth import HTTPBasicAuth\n",
|
||||
"import time\n",
|
||||
"import datetime"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"module_name = \"pu-mgalynchik-pa-mm25-synth\"\n",
|
||||
"username = \"developer\"\n",
|
||||
"password = \"WpNJxt7vCdP4Q9TYFX5Sh6ukGZHyrD\"\n",
|
||||
"REQUESTS_TIMEOUT = 20\n",
|
||||
"\n",
|
||||
"basic_auth = HTTPBasicAuth(username, password)\n",
|
||||
"\n",
|
||||
"headers = {\n",
|
||||
" \"Content-Type\": \"application/json\"\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"service_url = f\"https://platform-dev.stratpro.hse.ru/{module_name}\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Путь к файлу с входными данными, как локально, так и в файловом хранилище фреймворка\n",
|
||||
"filename = \"data2/telecom/telecom_data.csv\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'name': 'pu-mgalynchik-pa-mm25-synth/files/synthdata-box/data2/telecom/telecom_data.csv',\n",
|
||||
" 'presigned_put_url': 'https://storage.yandexcloud.net/unip-bucket/pu-mgalynchik-pa-mm25-synth/synthdata-box/users/developer/file_groups/data2/telecom/telecom_data.csv?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=YCAJEkP1cFQyhqJlPhj7QWi-9%2F20241220%2Fru-central1%2Fs3%2Faws4_request&X-Amz-Date=20241220T195650Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=2c8ad0adc28eeda89d2d2274971d6051fd8894a5dd32d1caa55b11b1777948ba'}"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"response = requests.put(f\"{service_url}/files/synthdata-box/{filename}\", auth=basic_auth)\n",
|
||||
"if response.status_code in [400, ]:\n",
|
||||
" response = requests.get(f\"{service_url}/files/synthdata-box/{filename}\", auth=basic_auth)\n",
|
||||
"display(response.json())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"200\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"if response.status_code in [200, 201]:\n",
|
||||
" result_urls = json.loads(response.content) \n",
|
||||
" \n",
|
||||
" with open(filename, \"rb\") as f:\n",
|
||||
" response = requests.put(result_urls[\"presigned_put_url\"], data=f.read())\n",
|
||||
" print(response.status_code)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"##### Запуск пайплайна генерации данных"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"pipeline_name = \"train-generate\"\n",
|
||||
"full_url = f\"https://platform-dev.stratpro.hse.ru/{module_name}/pipelines/{pipeline_name}/trials\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"synthdata_request = {\n",
|
||||
" \"inputs\": [\n",
|
||||
" {\n",
|
||||
" \"name\": \"input_data\",\n",
|
||||
" \"datatype\": \"FILE\",\n",
|
||||
" \"content_type\": \"text/csv\",\n",
|
||||
" \"shape\": [7043, 20],\n",
|
||||
" \"data\": \"data2/telecom/\"\n",
|
||||
" }\n",
|
||||
" ],\n",
|
||||
" \"output_vars\": [\n",
|
||||
" {\n",
|
||||
" \"name\": \"report_file\",\n",
|
||||
" \"data\": \"results2/syntesize_data/report/\"\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"name\": \"output_data\",\n",
|
||||
" \"data\": \"results2/syntesize_data/output/\"\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"name\": \"model\",\n",
|
||||
" \"data\": \"results2/syntesize_data/model/\"\n",
|
||||
" },\n",
|
||||
" ]\n",
|
||||
"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'https://platform-dev.stratpro.hse.ru/pu-mgalynchik-pa-mm25-synth/trials/4c4b3d7d-d960-4c1d-ad31-c35b1f5f1744'"
|
||||
]
|
||||
},
|
||||
"execution_count": 14,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"tracking_url"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'app_name': 'pu-mgalynchik-pa-mm25-synth',\n",
|
||||
" 'user_id': 'developer',\n",
|
||||
" 'tracking_id': '4c4b3d7d-d960-4c1d-ad31-c35b1f5f1744',\n",
|
||||
" 'pipeline_name': 'train-generate',\n",
|
||||
" 'pipeline_version': '34929876',\n",
|
||||
" 'next_pipeline_name': None,\n",
|
||||
" 'next_tracking_id': None,\n",
|
||||
" 'vars': {'inputs': [{'name': 'input_data',\n",
|
||||
" 'box_name': 'synthdata-box',\n",
|
||||
" 'data': 'data2/telecom/',\n",
|
||||
" 'datatype': 'FILE',\n",
|
||||
" 'content_type': 'text/csv'}],\n",
|
||||
" 'outputs': [{'name': 'report_file',\n",
|
||||
" 'box_name': 'synthdata-box',\n",
|
||||
" 'data': 'results2/syntesize_data/report/',\n",
|
||||
" 'datatype': 'FILE',\n",
|
||||
" 'content_type': None},\n",
|
||||
" {'name': 'output_data',\n",
|
||||
" 'box_name': 'synthdata-box',\n",
|
||||
" 'data': 'results2/syntesize_data/output/',\n",
|
||||
" 'datatype': 'FILE',\n",
|
||||
" 'content_type': None},\n",
|
||||
" {'name': 'model',\n",
|
||||
" 'box_name': 'synthdata-box',\n",
|
||||
" 'data': 'results2/syntesize_data/model/',\n",
|
||||
" 'datatype': 'FILE',\n",
|
||||
" 'content_type': None}]},\n",
|
||||
" 'status': {'conditions': [{'type': 'Started',\n",
|
||||
" 'condition_status': 'True',\n",
|
||||
" 'last_transition_time': '2024-12-20T19:57:04.310958+0000',\n",
|
||||
" 'message': '',\n",
|
||||
" 'reason': '',\n",
|
||||
" 'stage': None}],\n",
|
||||
" 'status_date_time': '2024-12-20T19:57:04.310958+0000'},\n",
|
||||
" '_links': {'self': {'href': 'https://platform-dev.stratpro.hse.ru/pu-mgalynchik-pa-mm25-synth/trials/4c4b3d7d-d960-4c1d-ad31-c35b1f5f1744',\n",
|
||||
" 'templated': None},\n",
|
||||
" 'pipeline': {'href': 'https://platform-dev.stratpro.hse.ru/pu-mgalynchik-pa-mm25-synth/pipelines/train-generate',\n",
|
||||
" 'templated': None},\n",
|
||||
" 'next_trial': None}}"
|
||||
]
|
||||
},
|
||||
"execution_count": 13,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"response.json()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"<Response [503]>"
|
||||
]
|
||||
},
|
||||
"execution_count": 18,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"tracking_info"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 19,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[0:00:00] Pipeline job status - Started\n",
|
||||
"[0:02:29] Pipeline job status - OutputsAreValidated\n",
|
||||
"[0:02:32] Pipeline job status - Completed\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'type': 'Completed',\n",
|
||||
" 'condition_status': 'True',\n",
|
||||
" 'last_transition_time': '2024-12-20T19:59:36.835439+0000',\n",
|
||||
" 'message': '',\n",
|
||||
" 'reason': '',\n",
|
||||
" 'stage': None}"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"tracking_info = requests.get(tracking_url, auth=basic_auth)\n",
|
||||
"last_status = json.loads(tracking_info.content)['status']['conditions'][-1]\n",
|
||||
"status_history = [last_status]\n",
|
||||
"while last_status['type'] != 'Completed':\n",
|
||||
" if last_status not in status_history:\n",
|
||||
" status_history.append(last_status)\n",
|
||||
" \n",
|
||||
" time.sleep(REQUESTS_TIMEOUT)\n",
|
||||
" \n",
|
||||
" tracking_info = requests.get(tracking_url, auth=basic_auth)\n",
|
||||
" last_status = json.loads(tracking_info.content)['status']['conditions'][-1]\n",
|
||||
"\n",
|
||||
"status_conditions = json.loads(tracking_info.content)['status']['conditions']\n",
|
||||
"start_time = datetime.datetime.strptime(status_conditions[0]['last_transition_time'][:-5], '%Y-%m-%dT%H:%M:%S.%f')\n",
|
||||
"for idx, status in enumerate(status_conditions):\n",
|
||||
" status_time = datetime.datetime.strptime(status['last_transition_time'][:-5], '%Y-%m-%dT%H:%M:%S.%f')\n",
|
||||
" print(f\"[{str(status_time - start_time).split('.', maxsplit=1)[0]}] Pipeline job status - {status['type']}\")\n",
|
||||
"\n",
|
||||
"display(last_status)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "JSONDecodeError",
|
||||
"evalue": "Expecting value: line 1 column 1 (char 0)",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mJSONDecodeError\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[0;32mIn[8], line 6\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m response\u001b[38;5;241m.\u001b[39mstatus_code \u001b[38;5;129;01min\u001b[39;00m [\u001b[38;5;241m200\u001b[39m, ]:\n\u001b[1;32m 5\u001b[0m tracking_info \u001b[38;5;241m=\u001b[39m requests\u001b[38;5;241m.\u001b[39mget(tracking_url, auth\u001b[38;5;241m=\u001b[39mbasic_auth)\n\u001b[0;32m----> 6\u001b[0m last_status \u001b[38;5;241m=\u001b[39m \u001b[43mjson\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mloads\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtracking_info\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcontent\u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mstatus\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mconditions\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]\n\u001b[1;32m 7\u001b[0m status_history \u001b[38;5;241m=\u001b[39m [last_status]\n\u001b[1;32m 8\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m last_status[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtype\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mCompleted\u001b[39m\u001b[38;5;124m'\u001b[39m:\n",
|
||||
"File \u001b[0;32m/usr/lib/python3.10/json/__init__.py:346\u001b[0m, in \u001b[0;36mloads\u001b[0;34m(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m 341\u001b[0m s \u001b[38;5;241m=\u001b[39m s\u001b[38;5;241m.\u001b[39mdecode(detect_encoding(s), \u001b[38;5;124m'\u001b[39m\u001b[38;5;124msurrogatepass\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 343\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m object_hook \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m\n\u001b[1;32m 344\u001b[0m parse_int \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m parse_float \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m\n\u001b[1;32m 345\u001b[0m parse_constant \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m object_pairs_hook \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m kw):\n\u001b[0;32m--> 346\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_default_decoder\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 347\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 348\u001b[0m \u001b[38;5;28mcls\u001b[39m \u001b[38;5;241m=\u001b[39m JSONDecoder\n",
|
||||
"File \u001b[0;32m/usr/lib/python3.10/json/decoder.py:337\u001b[0m, in \u001b[0;36mJSONDecoder.decode\u001b[0;34m(self, s, _w)\u001b[0m\n\u001b[1;32m 332\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecode\u001b[39m(\u001b[38;5;28mself\u001b[39m, s, _w\u001b[38;5;241m=\u001b[39mWHITESPACE\u001b[38;5;241m.\u001b[39mmatch):\n\u001b[1;32m 333\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Return the Python representation of ``s`` (a ``str`` instance\u001b[39;00m\n\u001b[1;32m 334\u001b[0m \u001b[38;5;124;03m containing a JSON document).\u001b[39;00m\n\u001b[1;32m 335\u001b[0m \n\u001b[1;32m 336\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 337\u001b[0m obj, end \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mraw_decode\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43midx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_w\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mend\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 338\u001b[0m end \u001b[38;5;241m=\u001b[39m _w(s, end)\u001b[38;5;241m.\u001b[39mend()\n\u001b[1;32m 339\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m end \u001b[38;5;241m!=\u001b[39m \u001b[38;5;28mlen\u001b[39m(s):\n",
|
||||
"File \u001b[0;32m/usr/lib/python3.10/json/decoder.py:355\u001b[0m, in \u001b[0;36mJSONDecoder.raw_decode\u001b[0;34m(self, s, idx)\u001b[0m\n\u001b[1;32m 353\u001b[0m obj, end \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mscan_once(s, idx)\n\u001b[1;32m 354\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[0;32m--> 355\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m JSONDecodeError(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mExpecting value\u001b[39m\u001b[38;5;124m\"\u001b[39m, s, err\u001b[38;5;241m.\u001b[39mvalue) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 356\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m obj, end\n",
|
||||
"\u001b[0;31mJSONDecodeError\u001b[0m: Expecting value: line 1 column 1 (char 0)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"response = requests.post(full_url, headers=headers, auth=basic_auth, json=synthdata_request)\n",
|
||||
"tracking_url = json.loads(response.content)['_links']['self']['href']\n",
|
||||
"\n",
|
||||
"if response.status_code in [200, ]:\n",
|
||||
" tracking_info = requests.get(tracking_url, auth=basic_auth)\n",
|
||||
" last_status = json.loads(tracking_info.content)['status']['conditions'][-1]\n",
|
||||
" status_history = [last_status]\n",
|
||||
" while last_status['type'] != 'Completed':\n",
|
||||
" if last_status not in status_history:\n",
|
||||
" status_history.append(last_status)\n",
|
||||
" \n",
|
||||
" time.sleep(REQUESTS_TIMEOUT)\n",
|
||||
" \n",
|
||||
" tracking_info = requests.get(tracking_url, auth=basic_auth)\n",
|
||||
" last_status = json.loads(tracking_info.content)['status']['conditions'][-1]\n",
|
||||
"\n",
|
||||
" status_conditions = json.loads(tracking_info.content)['status']['conditions']\n",
|
||||
" start_time = datetime.datetime.strptime(status_conditions[0]['last_transition_time'][:-5], '%Y-%m-%dT%H:%M:%S.%f')\n",
|
||||
" for idx, status in enumerate(status_conditions):\n",
|
||||
" status_time = datetime.datetime.strptime(status['last_transition_time'][:-5], '%Y-%m-%dT%H:%M:%S.%f')\n",
|
||||
" print(f\"[{str(status_time - start_time).split('.', maxsplit=1)[0]}] Pipeline job status - {status['type']}\")\n",
|
||||
" \n",
|
||||
" display(last_status)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"##### Результаты"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'name': 'pu-mgalynchik-pa-mm25-synth/files/synthdata-box/results2/syntesize_data/report/',\n",
|
||||
" 'files': [{'name': 'pu-mgalynchik-pa-mm25-synth/files/synthdata-box/results2/syntesize_data/report/report.json',\n",
|
||||
" 'presigned_get_url': 'https://storage.yandexcloud.net/unip-bucket/pu-mgalynchik-pa-mm25-synth/synthdata-box/users/developer/file_groups/results2/syntesize_data/report/report.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=YCAJEkP1cFQyhqJlPhj7QWi-9%2F20241220%2Fru-central1%2Fs3%2Faws4_request&X-Amz-Date=20241220T200128Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=6d3f3dfae9d69d7b73f82876dfde6626c5448851722c1deac0abdfd7e12a6105'}]}"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'name': 'pu-mgalynchik-pa-mm25-synth/files/synthdata-box/results2/syntesize_data/output/',\n",
|
||||
" 'files': [{'name': 'pu-mgalynchik-pa-mm25-synth/files/synthdata-box/results2/syntesize_data/output/prediction.csv',\n",
|
||||
" 'presigned_get_url': 'https://storage.yandexcloud.net/unip-bucket/pu-mgalynchik-pa-mm25-synth/synthdata-box/users/developer/file_groups/results2/syntesize_data/output/prediction.csv?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=YCAJEkP1cFQyhqJlPhj7QWi-9%2F20241220%2Fru-central1%2Fs3%2Faws4_request&X-Amz-Date=20241220T200128Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=c70d7ebbd679cc2ed83390911d8578237459d38f78ed4b2d05ef951f8b1a1c20'}]}"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'name': 'pu-mgalynchik-pa-mm25-synth/files/synthdata-box/results2/syntesize_data/model/',\n",
|
||||
" 'files': [{'name': 'pu-mgalynchik-pa-mm25-synth/files/synthdata-box/results2/syntesize_data/model/model.joblib',\n",
|
||||
" 'presigned_get_url': 'https://storage.yandexcloud.net/unip-bucket/pu-mgalynchik-pa-mm25-synth/synthdata-box/users/developer/file_groups/results2/syntesize_data/model/model.joblib?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=YCAJEkP1cFQyhqJlPhj7QWi-9%2F20241220%2Fru-central1%2Fs3%2Faws4_request&X-Amz-Date=20241220T200128Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=98993f950c6295a422dde2b2f46ed6c996bd565c55c8f90777bf48c9001094d6'}]}"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"for filename in synthdata_request[\"output_vars\"]:\n",
|
||||
" response = requests.get(f\"{service_url}/files/synthdata-box/{filename['data']}\", auth=basic_auth)\n",
|
||||
" if response.status_code in [200, 201]:\n",
|
||||
" display(response.json())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"##### Процент совпадений с входными данными:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'Overall Quality Score': 0.9492168339013363}"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"report_fname = synthdata_request[\"output_vars\"][0]['data']\n",
|
||||
"response = requests.get(f\"{service_url}/files/synthdata-box/{report_fname}\", auth=basic_auth)\n",
|
||||
"if response.status_code in [200, 201]:\n",
|
||||
" response = requests.get(response.json()['files'][0]['presigned_get_url'], auth=basic_auth)\n",
|
||||
" display(response.json())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.12"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
183
mm25-module-card.md
Normal file
183
mm25-module-card.md
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
### Проверка работы с модулем 25 через API
|
||||
|
||||
#### Обучение модели и генерация данных
|
||||
|
||||
##### Файловый API
|
||||
|
||||
|
||||
```python
|
||||
import requests
|
||||
import json
|
||||
from requests.auth import HTTPBasicAuth
|
||||
import time
|
||||
import datetime
|
||||
```
|
||||
|
||||
|
||||
```python
|
||||
module_name = "pu-mgalynchik-pa-mm25-synthdata"
|
||||
username = ""
|
||||
password = ""
|
||||
REQUESTS_TIMEOUT = 20
|
||||
|
||||
basic_auth = HTTPBasicAuth(username, password)
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
service_url = f"https://platform-dev-cs-hse.objectoriented.ru/{module_name}"
|
||||
```
|
||||
|
||||
|
||||
```python
|
||||
# Путь к файлу с входными данными, как локально, так и в файловом хранилище фреймворка
|
||||
filename = "data/telecom_data.csv"
|
||||
```
|
||||
|
||||
|
||||
```python
|
||||
response = requests.put(f"{service_url}/files/synthdata-box/{filename}", auth=basic_auth)
|
||||
if response.status_code in [400, ]:
|
||||
response = requests.get(f"{service_url}/files/synthdata-box/{filename}", auth=basic_auth)
|
||||
display(response.json())
|
||||
```
|
||||
|
||||
|
||||
{'name': 'pu-mgalynchik-pa-mm25-synthdata/files/synthdata-box/data/telecom_data.csv',
|
||||
'presigned_put_url': 'https://storage.yandexcloud.net/platform-default-user-data/pu-mgalynchik-pa-mm25-synthdata/synthdata-box/users/developer/file_groups/data/telecom_data.csv?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=YCAJEw7KWeJzGZz9pXuFdhLPP%2F20241201%2Fru-central1%2Fs3%2Faws4_request&X-Amz-Date=20241201T185235Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=e816ff92cb18b85f8f65e6c6536f9757271e12f28afb2c1dfc12616b462ce640'}
|
||||
|
||||
|
||||
|
||||
```python
|
||||
if response.status_code in [200, 201]:
|
||||
result_urls = json.loads(response.content)
|
||||
|
||||
with open(filename, "rb") as f:
|
||||
response = requests.put(result_urls["presigned_put_url"], data=f.read())
|
||||
print(response.status_code)
|
||||
```
|
||||
|
||||
200
|
||||
|
||||
|
||||
##### Запуск пайплайна генерации данных
|
||||
|
||||
|
||||
```python
|
||||
pipeline_name = "train-generate"
|
||||
full_url = f"https://platform-dev-cs-hse.objectoriented.ru/{module_name}/pipelines/{pipeline_name}/trials"
|
||||
```
|
||||
|
||||
|
||||
```python
|
||||
synthdata_request = {
|
||||
"inputs": [
|
||||
{
|
||||
"name": "input_data",
|
||||
"datatype": "FILE",
|
||||
"content_type": "text/csv",
|
||||
"shape": [7043, 20],
|
||||
"data": "data/"
|
||||
}
|
||||
],
|
||||
"output_vars": [
|
||||
{
|
||||
"name": "report_file",
|
||||
"data": "results/syntesize_data/report/"
|
||||
},
|
||||
{
|
||||
"name": "output_data",
|
||||
"data": "results/syntesize_data/output/"
|
||||
},
|
||||
{
|
||||
"name": "model",
|
||||
"data": "results/syntesize_data/model/"
|
||||
},
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
```python
|
||||
response = requests.post(full_url, headers=headers, auth=basic_auth, json=synthdata_request)
|
||||
tracking_url = json.loads(response.content)['_links']['self']['href']
|
||||
|
||||
if response.status_code in [200, ]:
|
||||
tracking_info = requests.get(tracking_url, auth=basic_auth)
|
||||
last_status = json.loads(tracking_info.content)['status']['conditions'][-1]
|
||||
status_history = [last_status]
|
||||
while last_status['type'] != 'Completed':
|
||||
if last_status not in status_history:
|
||||
status_history.append(last_status)
|
||||
|
||||
time.sleep(REQUESTS_TIMEOUT)
|
||||
|
||||
tracking_info = requests.get(tracking_url, auth=basic_auth)
|
||||
last_status = json.loads(tracking_info.content)['status']['conditions'][-1]
|
||||
|
||||
status_conditions = json.loads(tracking_info.content)['status']['conditions']
|
||||
start_time = datetime.datetime.strptime(status_conditions[0]['last_transition_time'][:-5], '%Y-%m-%dT%H:%M:%S.%f')
|
||||
for idx, status in enumerate(status_conditions):
|
||||
status_time = datetime.datetime.strptime(status['last_transition_time'][:-5], '%Y-%m-%dT%H:%M:%S.%f')
|
||||
print(f"[{str(status_time - start_time).split('.', maxsplit=1)[0]}] Pipeline job status - {status['type']}")
|
||||
|
||||
display(last_status)
|
||||
```
|
||||
|
||||
[0:00:00] Pipeline job status - Started
|
||||
[0:02:34] Pipeline job status - OutputsAreValidated
|
||||
[0:02:36] Pipeline job status - Completed
|
||||
|
||||
|
||||
|
||||
{'type': 'Completed',
|
||||
'condition_status': 'True',
|
||||
'last_transition_time': '2024-12-01T18:55:13.943784+0000',
|
||||
'message': '',
|
||||
'reason': '',
|
||||
'stage': None}
|
||||
|
||||
|
||||
##### Результаты
|
||||
|
||||
|
||||
```python
|
||||
for filename in synthdata_request["output_vars"]:
|
||||
response = requests.get(f"{service_url}/files/synthdata-box/{filename['data']}", auth=basic_auth)
|
||||
if response.status_code in [200, 201]:
|
||||
display(response.json())
|
||||
```
|
||||
|
||||
|
||||
{'name': 'pu-mgalynchik-pa-mm25-synthdata/files/synthdata-box/results/syntesize_data/report/',
|
||||
'files': [{'name': 'pu-mgalynchik-pa-mm25-synthdata/files/synthdata-box/results/syntesize_data/report/report.json',
|
||||
'presigned_get_url': 'https://storage.yandexcloud.net/platform-default-user-data/pu-mgalynchik-pa-mm25-synthdata/synthdata-box/users/developer/file_groups/results/syntesize_data/report/report.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=YCAJEw7KWeJzGZz9pXuFdhLPP%2F20241201%2Fru-central1%2Fs3%2Faws4_request&X-Amz-Date=20241201T185519Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=58608407ac224691cacd571fe7f5edb3b80ccb1fbb25632b2d3cad6be541d049'}]}
|
||||
|
||||
|
||||
|
||||
{'name': 'pu-mgalynchik-pa-mm25-synthdata/files/synthdata-box/results/syntesize_data/output/',
|
||||
'files': [{'name': 'pu-mgalynchik-pa-mm25-synthdata/files/synthdata-box/results/syntesize_data/output/prediction.csv',
|
||||
'presigned_get_url': 'https://storage.yandexcloud.net/platform-default-user-data/pu-mgalynchik-pa-mm25-synthdata/synthdata-box/users/developer/file_groups/results/syntesize_data/output/prediction.csv?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=YCAJEw7KWeJzGZz9pXuFdhLPP%2F20241201%2Fru-central1%2Fs3%2Faws4_request&X-Amz-Date=20241201T185519Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=5346314f79abfb31563be22b814a1286035cbb0d19f782f626166651b5691036'}]}
|
||||
|
||||
|
||||
|
||||
{'name': 'pu-mgalynchik-pa-mm25-synthdata/files/synthdata-box/results/syntesize_data/model/',
|
||||
'files': [{'name': 'pu-mgalynchik-pa-mm25-synthdata/files/synthdata-box/results/syntesize_data/model/model.joblib',
|
||||
'presigned_get_url': 'https://storage.yandexcloud.net/platform-default-user-data/pu-mgalynchik-pa-mm25-synthdata/synthdata-box/users/developer/file_groups/results/syntesize_data/model/model.joblib?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=YCAJEw7KWeJzGZz9pXuFdhLPP%2F20241201%2Fru-central1%2Fs3%2Faws4_request&X-Amz-Date=20241201T185519Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=41d8ded4948d817538aa65a12b6cfc028d7df5e0af963e5ce7c84d509c458b06'}]}
|
||||
|
||||
|
||||
##### Процент совпадений с входными данными:
|
||||
|
||||
|
||||
```python
|
||||
report_fname = synthdata_request["output_vars"][0]['data']
|
||||
response = requests.get(f"{service_url}/files/synthdata-box/{report_fname}", auth=basic_auth)
|
||||
if response.status_code in [200, 201]:
|
||||
response = requests.get(response.json()['files'][0]['presigned_get_url'], auth=basic_auth)
|
||||
display(response.json())
|
||||
```
|
||||
|
||||
|
||||
{'Overall Quality Score': 0.9413369301611234}
|
||||
|
||||
3
results/report/report.json
Normal file
3
results/report/report.json
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
"Overall Quality Score": 0.9456371017271323
|
||||
}
|
||||
Loading…
Reference in a new issue