import torch
import os
import pandas as pd
import numpy as np
import pytorch_lightning as pl

from decouple import config
from torch.utils.data import Dataset, DataLoader
from transformers import BertForSequenceClassification
from transformers import AutoTokenizer
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from torch.optim import AdamW
from datetime import datetime
from typing import Any

# atribui valores padrão
DEFAULTS: dict[str, Any] = {
    'seed': config('SEED', cast=int, default=42),
    'padding': config('PADDING', default='max_length', cast=str),
    'truncation': config('TRUNCATION', default=True, cast=bool),
    'truncation_side': config('TRUNCATION_SIDE', default='right', cast=str),
    'return_tensors': config('RETURN_TENSORS', default='pt', cast=str),
    'max_tokens': config('MAX_TOKENS', default=512, cast=int),
    'batch_size': config('BATCH_SIZE', default=16, cast=int),
    'num_workers': config('MAX_TOKENS', default=512, cast=int),
    'split_size': config('SPLIT_SIZE_BERT', default=.2, cast=float),
    'learning_rate': config('LEARNING_RATE_BERT', default=2e-5, cast=float),
    'max_epochs': config('MAX_EPOCHS', default=5, cast=int),
    'enable_checkpointing': config('ENABLE_CHECKPOINTING', default=True, cast=bool),
}

# configure global seed
pl.seed_everything(DEFAULTS['seed'])

# helper functions
def get_variable_name(variable):
    for name in globals():
        if id(globals()[name]) == id(variable):
            return name
    for name in locals():
        if id(locals()[name]) == id(variable):
            return name
    return None

def check_possible_variable(variable, possible_values):
    if variable not in possible_values:
        raise ValueError(f"Only followed values are allowed for '{get_variable_name(variable)}': {possible_values}.")

def cria_logger(experimento: str, id: str):
    logger = pl.loggers.TensorBoardLogger(
        save_dir=os.getcwd(),
        name=experimento,
        version=id
    )
    return logger

class CreateDataset(Dataset):
    """
    Class that handles dataset, tokenizing and batching for BERT.
    Inherits from torch.utils.data.Dataset
    """

    def __init__(
            self,
            dataframe: pd.DataFrame,
            tokenizer,
            max_tokens: int,
            class_col: str,
            padding: str = DEFAULTS['padding'],
            truncation: bool = DEFAULTS['truncation'],
            truncation_side: str = DEFAULTS['truncation_side'],
            return_tensors: str = DEFAULTS['return_tensors'],
    ):

        check_possible_variable(
            truncation_side,
            ['left', 'right', 'combined']
        )

        self.dataframe = dataframe.copy()
        self.tokenizer = tokenizer
        self.max_tokens = max_tokens
        self.padding = padding
        self.class_col = class_col
        self.return_tensors = return_tensors
        self.truncation = truncation
        self.truncation_side = truncation_side

        if truncation_side == 'combined':
            self.truncation = False

        if type(tokenizer) == str:
            self.tokenizer = AutoTokenizer.from_pretrained(
                tokenizer,
                truncation_side=truncation_side
            )
        else:
            self.tokenizer = tokenizer

    def __len__(self):
        return len(self.dataframe)

    def __getitem__(self, index: int):

        conteudo = self.dataframe.iloc[index]['conteudo']
        label = self.dataframe.iloc[index][self.class_col]

        # tokenização
        encoding = self.tokenizer.encode_plus(
            conteudo,
            add_special_tokens=True,
            max_length=self.max_tokens,
            padding=self.padding,
            truncation=self.truncation,
            return_attention_mask=True,
            return_tensors=self.return_tensors
        )

        input_ids = encoding["input_ids"].flatten()
        attention_mask = encoding["attention_mask"].flatten()

        if self.truncation_side == 'combined':
            a = input_ids[0:255]
            c = input_ids[-1:]
            b = input_ids[-256:]
            input_ids = torch.cat([a, c, b])
            a = attention_mask[0:255]
            c = attention_mask[-1:]
            b = attention_mask[-256:]
            attention_mask = torch.cat([a, c, b])

        return dict(
            input_ids=input_ids,
            attention_mask=attention_mask,
            label=torch.tensor(label),
            conteudo=conteudo
        )

# noinspection PyPep8Naming
class DataModuler(pl.LightningDataModule):
    """Encapsulates all data loading logic and returns the necessary data loaders."""

    def __init__(
            self,
            X: np.ndarray,
            y: np.ndarray,
            tokenizer,
            split_size: float = DEFAULTS['split_size'],
            max_tokens: int = DEFAULTS['max_tokens'],
            batch_size: int = DEFAULTS['batch_size'],
            num_workers: int = DEFAULTS['num_workers'],
            padding: str = DEFAULTS['padding'],
            truncation: bool = DEFAULTS['truncation'],
            truncation_side: str = DEFAULTS['truncation_side']
    ):
        super().__init__()
        self.X = X
        self.y = y
        self.max_tokens = max_tokens
        self.batch_size = batch_size
        self.split_size = split_size
        self.num_workers = num_workers
        self.tokenizer = tokenizer
        self.padding = padding
        self.truncation = truncation
        self.truncation_side = truncation_side

        self.validation_dataframe = None
        self.train_dataframe = None
        self.train_dataset = None
        self.validation_dataset = None

    def prepare_data(self):
        print("Preparando dados...")
        print('...')

        # SPLIT
        # conteudo e classes
        X = self.X
        y = self.y

        print(f'Total de registros {len(X)}')
        print(f'Realizando split em {self.split_size * 100}%...')
        print('...')

        # split em conjunto de treinamento e conjunto de testes
        X_train, X_test, y_train, y_test = train_test_split(
            X,
            y,
            test_size=self.split_size,
            random_state=DEFAULTS['seed'],
            shuffle=True,
            stratify=y
        )

        # verificando tamanhos
        print(f'A base de treinamento possui {len(X_train)} exemplos')
        print(f'A base de validação possui {len(X_test)} exemplos')
        print('...')

        self.train_dataframe = pd.DataFrame(data={
            'conteudo': X_train.reshape(-1),
            'classe': y_train.reshape(-1)
        }
        )

        self.validation_dataframe = pd.DataFrame(data={
            'conteudo': X_test.reshape(-1),
            'classe': y_test.reshape(-1)
        }
        )

    def setup(self, stage=None):
        self.prepare_data()

        self.train_dataset = CreateDataset(
            dataframe=self.train_dataframe,
            tokenizer=self.tokenizer,
            max_tokens=self.max_tokens,
            class_col='classe',
            padding=self.padding,
            truncation=self.truncation,
            truncation_side=self.truncation_side
        )

        self.validation_dataset = CreateDataset(
            dataframe=self.validation_dataframe,
            tokenizer=self.tokenizer,
            max_tokens=self.max_tokens,
            class_col='classe',
            padding=self.padding,
            truncation=self.truncation,
            truncation_side=self.truncation_side
        )

        print('Datasets prontos!')
        print('')
        print('')

    def train_dataloader(self):
        return DataLoader(
            self.train_dataset,
            batch_size=self.batch_size,
            shuffle=True,
            num_workers=self.num_workers,
        )

    def val_dataloader(self):
        return DataLoader(
            self.validation_dataset,
            batch_size=self.batch_size,
            shuffle=False,
            num_workers=self.num_workers
        )

# noinspection PyTypeChecker
class BERTClassifier(pl.LightningModule):

    def __init__(
            self,
            model_name: str,
            num_classes: int,
            batch_size: int = DEFAULTS['batch_size'],
            freeze_bert: bool = False,
            lr: float = DEFAULTS['learning_rate'],
    ):
        super().__init__()

        self.bert = BertForSequenceClassification.from_pretrained(
            model_name,
            num_labels=num_classes,
            output_hidden_states=True,
            problem_type="single_label_classification"
        )

        self.batch_size = batch_size
        self.save_hyperparameters()
        self.lr = lr

        if freeze_bert:
            for param in self.bert.parameters():
                param.requires_grad = False

    def forward(self, input_ids, attention_mask):
        outputs = self.bert(
            input_ids=input_ids,
            attention_mask=attention_mask
        )
        logits = outputs.logits
        return logits

    def training_step(self, batch, batch_idx):

        input_ids = batch['input_ids']
        attention_mask = batch['attention_mask']
        labels = batch['label']

        output = self.bert(
            input_ids=input_ids,
            attention_mask=attention_mask,
            labels=labels
        )

        loss = output.loss

        log_dict = {
            'epoch': float(self.current_epoch) + 1,
            'train_loss': float(loss),
        }

        self.log_dict(log_dict, batch_size=self.batch_size)

        return loss

    def validation_step(self, batch, batch_idx):

        input_ids = batch['input_ids']
        attention_mask = batch['attention_mask']
        labels = batch['label']

        output = self.bert(
            input_ids=input_ids,
            attention_mask=attention_mask,
            # labels=labels
        )

        preds = torch.argmax(output.logits, dim=1)
        report = classification_report(
            labels.cpu(),
            preds.cpu(),
            output_dict=True,
            zero_division=0,
            digits=3
        )

        log_dict = {
            'epoch': float(self.current_epoch) + 1,
            'val_acc': report['accuracy'],
            'val_precision': report['macro avg']['precision'],
            'val_recall': report['macro avg']['recall'],
            'val_f1-score': report['macro avg']['f1-score'],
        }

        self.log_dict(log_dict, batch_size=self.batch_size)

    def configure_optimizers(self):
        optimizer = AdamW(
            self.parameters(),
            lr=self.lr
        )
        return optimizer

checkpoint_callback = pl.callbacks.ModelCheckpoint(
    verbose=True,
    monitor="val_f1-score",
    save_top_k=1,
    mode='max',
    save_on_train_epoch_end=False
)

# noinspection PyPep8Naming
def prepare_bert_objects(
        X,
        y,
        model_name: str,
        split_size: float = DEFAULTS['split_size'],
        max_tokens: int = DEFAULTS['max_tokens'],
        batch_size: int = DEFAULTS['batch_size'],
        lr: float = DEFAULTS['learning_rate'],
        id=None,
        experimento: str = "",
        tokenizer=None,
        fast_dev_run=False,  # recommended for debugging purposes
        num_workers=DEFAULTS['num_workers'],
        padding=DEFAULTS['padding'],
        truncation=DEFAULTS['truncation'],
        truncation_side=DEFAULTS['truncation_side'],
        devices="auto",
        max_epochs=DEFAULTS['max_epochs'],
        enable_checkpointing=DEFAULTS['enable_checkpointing'],
        callback=checkpoint_callback,
):

    num_classes = np.unique(y).shape[0]

    if id is None:
        now = datetime.now().strftime('%y-%m-%d_%H%M%S')
        id = f"{model_name.replace('/', '-')}__{max_tokens}__{truncation_side}__{now}"

    # modelo
    model = BERTClassifier(
        model_name=model_name,
        batch_size=batch_size,
        num_classes=num_classes,
        lr=lr
    )

    # data_moduler
    data_module = DataModuler(
        X,
        y,
        split_size=split_size,
        tokenizer=tokenizer,
        max_tokens=max_tokens,
        num_workers=num_workers,
        batch_size=batch_size,
        padding=padding,
        truncation=truncation,
        truncation_side=truncation_side
    )

    # trainer
    trainer = pl.Trainer(
        max_epochs=max_epochs,
        accelerator="auto",  # if not set is set to auto
        devices=devices,  # if not set is set to auto
        benchmark=True,  # can increase the speed of your system if your input sizes don’t change
        enable_checkpointing=enable_checkpointing,
        fast_dev_run=fast_dev_run,
        callbacks=[callback],
        logger=cria_logger(
            experimento=experimento,
            id=id
        ),
    )

    return model, data_module, trainer

def test_report(model, dataloader, device):
    """Use it to test the model over validation dataloader."""
    y_real = []
    y_preds = []

    total = len(dataloader)
    print(f"Total de {total} batches.")

    for i, batch in enumerate(dataloader):

        y_real.extend(batch['label'].tolist())

        logits = model(
            batch['input_ids'].to(device),
            batch['attention_mask'].to(device)
            )
        preds = torch.argmax(logits, dim=1).tolist()
        y_preds.extend(preds)

        if i % 200 == 0 and not i == 0:
            print(f"Batch {i}/{total} done....")

    print("Done!")
    print("")
    print("")
    print(classification_report(y_real, y_preds))
    report = classification_report(y_real, y_preds, output_dict=True, digits=3)

    return report, y_real, y_preds


import pandas as pd
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torchvision.transforms as transforms
from torch.utils.data import DataLoader, Dataset
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, confusion_matrix, recall_score, f1_score, classification_report
import matplotlib.pyplot as plt
import seaborn as sns
import os
from PIL import Image

# Define the CNN model
class CNNModel(nn.Module):
    def __init__(self):
        super(CNNModel, self).__init__()
        self.conv1 = nn.Conv2d(3, 16, kernel_size=3, padding=1)
        self.conv2 = nn.Conv2d(16, 32, kernel_size=3, padding=1)
        self.pool = nn.AdaptiveAvgPool2d(1)
        self.fc = nn.Linear(32, 1)

    def forward(self, x):
        x = nn.functional.relu(self.conv1(x))
        x = nn.functional.relu(self.conv2(x))
        x = self.pool(x)
        x = x.view(x.size(0), -1)
        x = torch.sigmoid(self.fc(x))
        return x

# Custom dataset class
class CustomDataset(Dataset):
    def __init__(self, image_folder, labels_data, transform=None):
        self.image_folder = image_folder
        self.labels_data = labels_data
        self.transform = transform

    def __len__(self):
        return len(self.labels_data)

    def __getitem__(self, idx):
        # img_name = os.path.join(self.image_folder, f'e{str(idx+1).zfill(5)}.png')
        img_name = os.path.join(self.image_folder, self.labels_data.iloc[idx, 0])

        image = Image.open(img_name).convert('RGB')
        label = self.labels_data.iloc[idx, 1]
        if self.transform:
            image = self.transform(image)
        return image, label

# Define transformations
transform = transforms.Compose([
    transforms.Resize((32, 32)),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
])

# raw data
# image_folder = 'image_cwt_raw/overall'
# labels_file = 'image_cwt_raw/overall_labels.csv'

image_folder = 'CWT_data_resampled/overall'
labels_file = 'CWT_data_resampled/overall_labels.csv'

# balanced data
# image_folder = 'CWT_data_resampled/training-e1'
# labels_file = 'CWT_data_resampled/training-e/e_labels.csv'

# Load and split the data
labels_data = pd.read_csv(labels_file)
train_data, test_data = train_test_split(labels_data, test_size=0.3, random_state=42)

train_dataset = CustomDataset(image_folder, train_data, transform=transform)
test_dataset = CustomDataset(image_folder, test_data, transform=transform)

train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
test_loader = DataLoader(test_dataset, batch_size=32, shuffle=False)

# Initialize the CNN model and optimizer
model = CNNModel()
criterion = nn.BCELoss()  # Binary Cross-Entropy loss
optimizer = optim.Adam(model.parameters(), lr=0.001)

# Train the model
num_epochs = 10
for epoch in range(num_epochs):
    for batch_X, batch_y in train_loader:
        optimizer.zero_grad()
        outputs = model(batch_X)
        loss = criterion(outputs, batch_y.unsqueeze(1).float())
        loss.backward()
        optimizer.step()

# Evaluate the model
model.eval()
with torch.no_grad():
    y_pred_probs = []
    y_true = []

    for batch_X, batch_y in train_loader:
        outputs = model(batch_X)
        y_pred_probs.extend(outputs)
        y_true.extend(batch_y)

y_pred_probs = torch.cat(y_pred_probs)
y_pred = (y_pred_probs >= 0.5).float()
y_true = torch.tensor(y_true)

# Calculate metrics
accuracy = accuracy_score(y_true, y_pred)
conf_matrix = confusion_matrix(y_true, y_pred)
tn, fp, fn, tp = conf_matrix.ravel()
sensitivity = tp / (tp + fn)
specificity = tn / (tn + fp)
uf1 = (2 * tp) / (2 * tp + fp + fn)
uar = (sensitivity + specificity) / 2

# Calculate the normalized confusion matrix
normalized_conf_matrix = conf_matrix.astype('float') / conf_matrix.sum(axis=1)[:, np.newaxis]

# Print metrics
classification_rep = classification_report(y_true, y_pred, target_names=['Class 0', 'Class 1'])
normalized_conf_matrix = conf_matrix.astype('float') / conf_matrix.sum(axis=1)[:, np.newaxis]

print("Accuracy:", accuracy)
print("Sensitivity:", sensitivity)
print("Specificity:", specificity)
print("Unweighted F1-Score (UF1):", uf1)
print("Unweighted Average Recall (UAR):", uar)

plt.rcParams['font.family'] = 'Times New Roman'
plt.rcParams['font.size'] = '46'
sns.set(font_scale=1.2)
sns.heatmap(normalized_conf_matrix, annot=True, cmap="Blues", fmt=".2f", xticklabels=['Normal', 'Abnormal'], yticklabels=['Normal', 'Abnormal'])

plt.xlabel('Predicted label')
plt.ylabel('True label')
plt.title('Normalised Confusion Matrix')
print("Classification Report:\n", classification_rep)

plt.show()


#!/usr/bin/env python3

from collections import OrderedDict
import logging
import os
from pathlib import Path
import shlex
import subprocess
import sys

from util import KernelVersion
from vmtest.config import (
    ARCHITECTURES,
    HOST_ARCHITECTURE,
    KERNEL_FLAVORS,
    SUPPORTED_KERNEL_VERSIONS,
    Kernel,
)
from vmtest.download import DownloadCompiler, DownloadKernel, download_in_thread
from vmtest.kmod import build_kmod
from vmtest.rootfsbuild import build_drgn_in_rootfs
from vmtest.vm import LostVMError, run_in_vm

logger = logging.getLogger(__name__)

class _ProgressPrinter:
    def __init__(self, file):
        self._file = file
        if hasattr(file, "fileno"):
            try:
                columns = os.get_terminal_size(file.fileno())[0]
                self._color = True
            except OSError:
                columns = 80
                self._color = False
        self._header = "#" * columns
        self._passed = {}
        self._failed = {}

    def _green(self, s: str) -> str:
        if self._color:
            return "\033[32m" + s + "\033[0m"
        else:
            return s

    def _red(self, s: str) -> str:
        if self._color:
            return "\033[31m" + s + "\033[0m"
        else:
            return s

    def update(self, category: str, name: str, passed: bool):
        d = self._passed if passed else self._failed
        d.setdefault(category, []).append(name)

        if self._failed:
            header = self._red(self._header)
        else:
            header = self._green(self._header)

        print(header, file=self._file)
        print(file=self._file)

        if self._passed:
            first = True
            for category, names in self._passed.items():
                if first:
                    first = False
                    print(self._green("Passed:"), end=" ")
                else:
                    print("       ", end=" ")
                print(f"{category}: {', '.join(names)}")
        if self._failed:
            first = True
            for category, names in self._failed.items():
                if first:
                    first = False
                    print(self._red("Failed:"), end=" ")
                else:
                    print("       ", end=" ")
                print(f"{category}: {', '.join(names)}")

        print(file=self._file)
        print(header, file=self._file, flush=True)

def _kdump_works(kernel: Kernel) -> bool:
    if kernel.arch.name == "aarch64":
        # kexec fails with "kexec: setup_2nd_dtb failed." on older versions.
        # See
        # http://lists.infradead.org/pipermail/kexec/2020-November/021740.html.
        return KernelVersion(kernel.release) >= KernelVersion("5.10")
    elif kernel.arch.name == "arm":
        # Without virtual address translation, we can't debug vmcores. Besides,
        # kexec fails with "Could not find a free area of memory of 0xXXX
        # bytes...".
        return False
    elif kernel.arch.name == "ppc64":
        # Before 6.1, sysrq-c hangs.
        return KernelVersion(kernel.release) >= KernelVersion("6.1")
    elif kernel.arch.name == "s390x":
        # Before 5.15, sysrq-c hangs.
        return KernelVersion(kernel.release) >= KernelVersion("5.15")
    elif kernel.arch.name == "x86_64":
        return True
    else:
        assert False, kernel.arch.name

if __name__ == "__main__":
    import argparse

    logging.basicConfig(
        format="%(asctime)s:%(levelname)s:%(name)s:%(message)s", level=logging.INFO
    )
    parser = argparse.ArgumentParser(
        description="test drgn in a virtual machine",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser.add_argument(
        "-d",
        "--directory",
        metavar="DIR",
        type=Path,
        default="build/vmtest",
        help="directory for vmtest artifacts",
    )
    parser.add_argument(
        "-a",
        "--architecture",
        dest="architectures",
        action="append",
        choices=["all", "foreign", *sorted(ARCHITECTURES)],
        default=argparse.SUPPRESS,
        required=HOST_ARCHITECTURE is None,
        help="architecture to test, "
        '"all" to test all supported architectures, '
        'or "foreign" to test all supported architectures other than the host architecture; '
        "may be given multiple times"
        + (
            "" if HOST_ARCHITECTURE is None else f" (default: {HOST_ARCHITECTURE.name})"
        ),
    )
    parser.add_argument(
        "-k",
        "--kernel",
        metavar="PATTERN|{all," + ",".join(KERNEL_FLAVORS) + "}",
        dest="kernels",
        action="append",
        default=argparse.SUPPRESS,
        help="kernel to test, "
        '"all" to test all supported kernels, '
        "or flavor name to test all supported kernels of a specific flavor; "
        "may be given multiple times (default: none)",
    )
    parser.add_argument(
        "-l",
        "--local",
        action="store_true",
        help="run local tests",
    )
    args = parser.parse_args()

    architecture_names = []
    if hasattr(args, "architectures"):
        for name in args.architectures:
            if name == "all":
                architecture_names.extend(ARCHITECTURES)
            elif name == "foreign":
                architecture_names.extend(
                    [
                        arch.name
                        for arch in ARCHITECTURES.values()
                        if arch is not HOST_ARCHITECTURE
                    ]
                )
            else:
                architecture_names.append(name)
        architectures = [
            ARCHITECTURES[name] for name in OrderedDict.fromkeys(architecture_names)
        ]
    else:
        architectures = [HOST_ARCHITECTURE]

    if hasattr(args, "kernels"):
        kernels = []
        for pattern in args.kernels:
            if pattern == "all":
                kernels.extend(
                    [
                        version + ".*" + flavor
                        for version in SUPPORTED_KERNEL_VERSIONS
                        for flavor in KERNEL_FLAVORS
                    ]
                )
            elif pattern in KERNEL_FLAVORS:
                kernels.extend(
                    [version + ".*" + pattern for version in SUPPORTED_KERNEL_VERSIONS]
                )
            else:
                kernels.append(pattern)
        args.kernels = OrderedDict.fromkeys(kernels)
    else:
        args.kernels = []

    if not args.kernels and not args.local:
        parser.error("at least one of -k/--kernel or -l/--local is required")

    if args.kernels:
        to_download = [DownloadCompiler(arch) for arch in architectures]
        for pattern in args.kernels:
            for arch in architectures:
                to_download.append(DownloadKernel(arch, pattern))
    else:
        to_download = []

    progress = _ProgressPrinter(sys.stderr)

    with download_in_thread(args.directory, to_download) as downloads:
        for arch in architectures:
            if arch is HOST_ARCHITECTURE:
                subprocess.check_call(
                    [sys.executable, "setup.py", "build_ext", "-i"],
                    env={
                        **os.environ,
                        "CONFIGURE_FLAGS": "--enable-compiler-warnings=error",
                    },
                )
                if args.local:
                    logger.info("running local tests on %s", arch.name)
                    status = subprocess.call(
                        [
                            sys.executable,
                            "-m",
                            "pytest",
                            "-v",
                            "--ignore=tests/linux_kernel",
                        ]
                    )
                    progress.update(arch.name, "local", status == 0)
            else:
                rootfs = args.directory / arch.name / "rootfs"
                build_drgn_in_rootfs(rootfs)
                if args.local:
                    logger.info("running local tests on %s", arch.name)
                    status = subprocess.call(
                        [
                            "unshare",
                            "--map-root-user",
                            "--map-users=auto",
                            "--map-groups=auto",
                            "--fork",
                            "--pid",
                            "--mount-proc=" + str(rootfs / "proc"),
                            "sh",
                            "-c",
                            r"""
set -e

mount --bind . "$1/mnt"
chroot "$1" sh -c 'cd /mnt && pytest -v --ignore=tests/linux_kernel'
""",
                            "sh",
                            rootfs,
                        ]
                    )
                    progress.update(arch.name, "local", status == 0)
        for kernel in downloads:
            if not isinstance(kernel, Kernel):
                continue
            kmod = build_kmod(args.directory, kernel)
            if _kdump_works(kernel):
                kdump_command = """\
    "$PYTHON" -Bm vmtest.enter_kdump
    # We should crash and not reach this.
    exit 1
"""
            else:
                kdump_command = ""
            test_command = rf"""
set -e

export PYTHON={shlex.quote(sys.executable)}
export DRGN_TEST_KMOD={shlex.quote(str(kmod))}
export DRGN_RUN_LINUX_KERNEL_TESTS=1
if [ -e /proc/vmcore ]; then
    "$PYTHON" -Bm pytest -v tests/linux_kernel/vmcore
else
    insmod "$DRGN_TEST_KMOD"
    "$PYTHON" -Bm pytest -v tests/linux_kernel --ignore=tests/linux_kernel/vmcore
{kdump_command}
fi
"""
            try:
                status = run_in_vm(
                    test_command,
                    kernel,
                    args.directory / kernel.arch.name / "rootfs",
                    args.directory,
                )
            except LostVMError as e:
                print("error:", e, file=sys.stderr)
                status = -1
            progress.update(kernel.arch.name, kernel.release, status == 0)


from tkinter import Frame, Label, Button, TOP, RIGHT, LEFT, CENTER, BOTTOM
from PIL import Image, ImageTk
import PIL
from Screens.shopButton import ShopButton
class StarShopEngine:
    def __init__(self, window):
        self.window = window
        self.normalShop = True
        self.f = Frame(self.window.root, bg="blue", width=self.window.width, height =self.window.height)
        self.f.pack_propagate(0)

        self.backgroundImage= ImageTk.PhotoImage(file="Backgrounds/starShop.png")
        self.backgroundLabel = Label(self.f, image=self.backgroundImage)

        self.shopScale = self.window.width / 1280
        self.save = self.window.save
        barScale = self.shopScale / 5
        buttonScale = self.shopScale / 2

        self.barImageList = []
        self.barImageList.append(self.openTkImage("Images/Shop/levelBar1.png", barScale))
        self.barImageList.append(self.openTkImage("Images/Shop/levelBar2.png", barScale))
        self.barImageList.append(self.openTkImage("Images/Shop/levelBar3.png", barScale))
        self.barImageList.append(self.openTkImage("Images/Shop/levelBar4.png", barScale))
        self.barImageList.append(self.openTkImage("Images/Shop/levelBar5.png", barScale))
        self.barImageList.append(self.openTkImage("Images/Shop/levelBar6.png", barScale))

        self.buttonList = []

        self.sideBar = Label(self.window.root, bg="#%02x%02x%02x" % (121, 202, 249), width=self.window.width // 5, height=self.window.height)
        self.sideBar.pack(in_=self.f, side=LEFT, pady=25, padx = 25)

        self.buttonGrid = Label(self.window.root, bg="#%02x%02x%02x" % (121, 202, 249), width=self.window.width // 5 * 3, height =self.window.height)
        self.buttonGrid.pack(in_=self.f, side=TOP)

        self.moneyL = Label(self.window.root, text="Stars: " + str(self.save.stars), bg="#%02x%02x%02x" % (121, 202, 249), font="Helvetica 15 bold", width=self.window.width // 5)
        self.moneyL.pack(side=TOP, in_=self.sideBar, pady=25)

        self.achievementB = Button(self.window.root, text="Shop", command=self.shop, bg="#%02x%02x%02x" % (255, 165, 0), width=self.window.width // 5, font="Helvetica 15 bold")
        self.achievementB.pack(side=BOTTOM, in_=self.sideBar, pady=25)

        self.accept = Button(self.window.root, text="Accept", command=self.accept, bg="#%02x%02x%02x" % (255, 0, 0), width=self.window.width // 5, font="Helvetica 15 bold")
        self.accept.pack(side=BOTTOM, in_=self.sideBar)

        self.shieldButtonImage = ImageTk.PhotoImage(self.scale(Image.open("Images/Shop/maxFuelButton.png"), buttonScale))
        ShopButton(self.window, self.window.save.starFuelCanLevel, self.buttonGrid, self.sideBar, self.save, self.buttonList, self.barImageList, self.backgroundImage, 0, 0,
                   """Fuel Cans:
Doubles the amount of fuel than fuel cans give you""",
                   self.window.root, image=self.shieldButtonImage)
        self.airResistButtonImage = ImageTk.PhotoImage(self.scale(Image.open("Images/Shop/propulsionButton.png"), buttonScale))
        ShopButton(self.window, self.window.save.starSpeedGateLevel, self.buttonGrid, self.sideBar, self.save, self.buttonList, self.barImageList, self.backgroundImage, 1, 0,
                   """Speed Gates:
Doubles the acceleration from speed gates""",
                   self.window.root, image=self.airResistButtonImage)
        self.antiGravButtonImage = ImageTk.PhotoImage(self.scale(Image.open("Images/Shop/shieldButton.png"), buttonScale))
        ShopButton(self.window, self.window.save.starShieldLevel, self.buttonGrid, self.sideBar, self.save, self.buttonList, self.barImageList, self.backgroundImage, 2, 0,
                   """Shields:
Reduces the amount of pull from black holes based on your number of shields""",
                   self.window.root, image=self.antiGravButtonImage)
        self.frictionButtonImage = ImageTk.PhotoImage(self.scale(Image.open("Images/Shop/magnetButton.png"), buttonScale))
        ShopButton(self.window, self.window.save.starMagnetLevel, self.buttonGrid, self.sideBar, self.save, self.buttonList, self.barImageList, self.backgroundImage, 0, 1,
                   """Magnet:
Gives you a magnet that sucks in blobs""",
                   self.window.root, image=self.frictionButtonImage)
        self.launchSpeedButtonImage = ImageTk.PhotoImage(self.scale(Image.open("Images/Shop/predictionButton.png"), buttonScale))
        ShopButton(self.window, self.window.save.starPredictionLevel, self.buttonGrid, self.sideBar, self.save, self.buttonList, self.barImageList, self.backgroundImage, 1, 1,
                   """Predictions:
Displays a line that shows the path you will travel along""",
                   self.window.root, image=self.launchSpeedButtonImage)
        self.maxMassButtonImage = ImageTk.PhotoImage(self.scale(Image.open("Images/Shop/blackHoleButton.png"), buttonScale))
        ShopButton(self.window, self.window.save.starBlackHoleLevel, self.buttonGrid, self.sideBar, self.save, self.buttonList, self.barImageList, self.backgroundImage, 2, 1,
                   """Black hole warnings:
Displays arrows that point towards black holes""",
                   self.window.root, image=self.maxMassButtonImage)

    def update(self):
        self.f.config(width=self.window.width, height=self.window.width)

        self.shopScale = self.window.width / 1280
        self.save = self.window.save
        barScale = self.shopScale / 5
        buttonScale = self.shopScale / 2

        self.sideBar.config(width=self.window.width // 5, heigh=self.window.height)
        self.buttonGrid.config(width=self.window.width // 5 * 3, height=self.window.height)

        self.barImageList.clear()
        self.barImageList.append(self.openTkImage("Images/Shop/levelBar1.png", barScale))
        self.barImageList.append(self.openTkImage("Images/Shop/levelBar2.png", barScale))
        self.barImageList.append(self.openTkImage("Images/Shop/levelBar3.png", barScale))
        self.barImageList.append(self.openTkImage("Images/Shop/levelBar4.png", barScale))
        self.barImageList.append(self.openTkImage("Images/Shop/levelBar5.png", barScale))
        self.barImageList.append(self.openTkImage("Images/Shop/levelBar6.png", barScale))

        self.buttonGrid.config(width=self.window.width // 5 * 3, height=self.window.height)
        try:
            self.shieldButtonImage = ImageTk.PhotoImage(self.scale(Image.open("Images/Shop/maxFuelButton.png"), buttonScale))
            self.buttonList[0].config(image=self.shieldButtonImage)
            self.airResistButtonImage = ImageTk.PhotoImage(self.scale(Image.open("Images/Shop/propulsionButton.png"), buttonScale))
            self.buttonList[1].config(image=self.airResistButtonImage)
            self.antiGravButtonImage = ImageTk.PhotoImage(self.scale(Image.open("Images/Shop/shieldButton.png"), buttonScale))
            self.buttonList[2].config(image=self.antiGravButtonImage)
            self.frictionButtonImage = ImageTk.PhotoImage(self.scale(Image.open("Images/Shop/magnetButton.png"), buttonScale))
            self.buttonList[3].config(image=self.frictionButtonImage)
            self.launchSpeedButtonImage = ImageTk.PhotoImage(self.scale(Image.open("Images/Shop/predictionButton.png"), buttonScale))
            self.buttonList[4].config(image=self.launchSpeedButtonImage)
            self.maxMassButtonImage = ImageTk.PhotoImage(self.scale(Image.open("Images/Shop/blackHoleButton.png"), buttonScale))
            self.buttonList[5].config(image=self.maxMassButtonImage)
        except:
            pass

    def accept(self):
        #must add in saving here, buttons dont currently do anything
        self.window.rMenu = "charScreen"

    def shop(self):
        self.window.rMenu = "shop"

    def run(self):
        #must save levels here
        for button in self.buttonList:
            button.run()
        self.save = self.window.save
        self.moneyL.config(text="Stars: " + str(int(self.save.stars)))
        for button in self.buttonList:
            button.save = self.window.save

        self.window.save.starFuelCanLevel = self.buttonList[0].level
        self.window.save.starSpeedGateLevel =self.buttonList[1].level
        self.window.save.starShieldLevel = self.buttonList[2].level
        self.window.save.starMagnetLevel = self.buttonList[3].level
        self.window.save.starPredictionLevel = self.buttonList[4].level
        self.window.save.starBlackHoleLevel = self.buttonList[5].level

    def scale(self, image, scale):
        newWidth = image.size[0] * scale
        wPercent = (newWidth/float(image.size[0]))
        hSize = int((float(image.size[1])*float(wPercent)))
        scaledImage = image.resize((int(newWidth), int(hSize)), PIL.Image.ANTIALIAS)
        return scaledImage

    def hide(self):
        self.backgroundLabel.place(x=10000, y=10000, relwidth=1, relheight=1)
        self.sideBar.place(x=10000, y=10000, relwidth=.15, relheight=1)
        self.f.pack_forget()

    def setUp(self):
        self.backgroundLabel.place(x=0, y=0, relwidth=1, relheight=1)
        self.sideBar.place(x=0, y=0, relwidth=.15, relheight=1)
        self.buttonList[0].level = self.window.save.starFuelCanLevel
        self.buttonList[1].level = self.window.save.starSpeedGateLevel
        self.buttonList[2].level = self.window.save.starShieldLevel
        self.buttonList[3].level = self.window.save.starMagnetLevel
        self.buttonList[4].level = self.window.save.starPredictionLevel
        self.buttonList[5].level = self.window.save.starBlackHoleLevel

        self.f.pack()

    def openTkImage(self, path, scale):
        return ImageTk.PhotoImage(self.scale(Image.open(path), scale).convert("RGB"))


import enum
import warnings
import weakref

from absl import logging
from dm_control import mjcf
from dm_control.composer import observation
from dm_control.rl import control
import dm_env
import numpy as np

warnings.simplefilter('always', DeprecationWarning)

_STEPS_LOGGING_INTERVAL = 10000

HOOK_NAMES = ('initialize_episode_mjcf',
              'after_compile',
              'initialize_episode',
              'before_step',
              'before_substep',
              'after_substep',
              'after_step')

_empty_function = lambda: None

def _empty_function_with_docstring():
  """Some docstring."""

_EMPTY_CODE = _empty_function.__code__.co_code
_EMPTY_WITH_DOCSTRING_CODE = _empty_function_with_docstring.__code__.co_code

def _callable_is_trivial(f):
  return (f.__code__.co_code == _EMPTY_CODE or
          f.__code__.co_code == _EMPTY_WITH_DOCSTRING_CODE)

class ObservationPadding(enum.Enum):
  INITIAL_VALUE = -1
  ZERO = 0

class EpisodeInitializationError(RuntimeError):
  """Raised by a `composer.Task` when it fails to initialize an episode."""

class _Hook:

  __slots__ = ('entity_hooks', 'extra_hooks')

  def __init__(self):
    self.entity_hooks = []
    self.extra_hooks = []

class _EnvironmentHooks:
  """Helper object that scans and memoizes various hooks in a task.

  This object exist to ensure that we do not incur a substantial overhead in
  calling empty entity hooks in more complicated tasks.
  """

  __slots__ = (('_task', '_episode_step_count') +
               tuple('_' + hook_name for hook_name in HOOK_NAMES))

  def __init__(self, task):
    self._task = task
    self._episode_step_count = 0
    for hook_name in HOOK_NAMES:
      slot_name = '_' + hook_name
      setattr(self, slot_name, _Hook())
    self.refresh_entity_hooks()

  def refresh_entity_hooks(self):
    """Scans and memoizes all non-trivial entity hooks."""
    for hook_name in HOOK_NAMES:
      hooks = []
      for entity in self._task.root_entity.iter_entities():
        entity_hook = getattr(entity, hook_name)
        # Ignore any hook that is a no-op to avoid function call overhead.
        if not _callable_is_trivial(entity_hook):
          hooks.append(entity_hook)
      getattr(self, '_' + hook_name).entity_hooks = hooks

  def add_extra_hook(self, hook_name, hook_callable):
    if hook_name not in HOOK_NAMES:
      raise ValueError('{!r} is not a valid hook name'.format(hook_name))
    if not callable(hook_callable):
      raise ValueError('{!r} is not a callable'.format(hook_callable))
    getattr(self, '_' + hook_name).extra_hooks.append(hook_callable)

  def initialize_episode_mjcf(self, random_state):
    self._task.initialize_episode_mjcf(random_state)
    for entity_hook in self._initialize_episode_mjcf.entity_hooks:
      entity_hook(random_state)
    for extra_hook in self._initialize_episode_mjcf.extra_hooks:
      extra_hook(random_state)

  def after_compile(self, physics, random_state):
    self._task.after_compile(physics, random_state)
    for entity_hook in self._after_compile.entity_hooks:
      entity_hook(physics, random_state)
    for extra_hook in self._after_compile.extra_hooks:
      extra_hook(physics, random_state)

  def initialize_episode(self, physics, random_state):
    self._episode_step_count = 0
    self._task.initialize_episode(physics, random_state)
    for entity_hook in self._initialize_episode.entity_hooks:
      entity_hook(physics, random_state)
    for extra_hook in self._initialize_episode.extra_hooks:
      extra_hook(physics, random_state)

  def before_step(self, physics, action, random_state):
    self._episode_step_count += 1
    if self._episode_step_count % _STEPS_LOGGING_INTERVAL == 0:
      logging.info('The current episode has been running for %d steps.',
                   self._episode_step_count)
    self._task.before_step(physics, action, random_state)
    for entity_hook in self._before_step.entity_hooks:
      entity_hook(physics, random_state)
    for extra_hook in self._before_step.extra_hooks:
      extra_hook(physics, action, random_state)

  def before_substep(self, physics, action, random_state):
    self._task.before_substep(physics, action, random_state)
    for entity_hook in self._before_substep.entity_hooks:
      entity_hook(physics, random_state)
    for extra_hooks in self._before_substep.extra_hooks:
      extra_hooks(physics, action, random_state)

  def after_substep(self, physics, random_state):
    self._task.after_substep(physics, random_state)
    for entity_hook in self._after_substep.entity_hooks:
      entity_hook(physics, random_state)
    for extra_hook in self._after_substep.extra_hooks:
      extra_hook(physics, random_state)

  def after_step(self, physics, random_state):
    self._task.after_step(physics, random_state)
    for entity_hook in self._after_step.entity_hooks:
      entity_hook(physics, random_state)
    for extra_hook in self._after_step.extra_hooks:
      extra_hook(physics, random_state)

class _CommonEnvironment:
  """Common components for RL environments."""

  def __init__(self, task, time_limit=float('inf'), random_state=None,
               n_sub_steps=None,
               raise_exception_on_physics_error=True,
               strip_singleton_obs_buffer_dim=False,
               delayed_observation_padding=ObservationPadding.ZERO,
               legacy_step: bool = True):
    """Initializes an instance of `_CommonEnvironment`.

    Args:
      task: Instance of `composer.base.Task`.
      time_limit: (optional) A float, the time limit in seconds beyond which an
        episode is forced to terminate.
      random_state: Optional, either an int seed or an `np.random.RandomState`
        object. If None (default), the random number generator will self-seed
        from a platform-dependent source of entropy.
      n_sub_steps: (DEPRECATED) An integer, number of physics steps to take per
        agent control step. New code should instead override the
        `control_substep` property of the task.
      raise_exception_on_physics_error: (optional) A boolean, indicating whether
        `PhysicsError` should be raised as an exception. If `False`, physics
        errors will result in the current episode being terminated with a
        warning logged, and a new episode started.
      strip_singleton_obs_buffer_dim: (optional) A boolean, if `True`,
        the array shape of observations with `buffer_size == 1` will not have a
        leading buffer dimension.
      delayed_observation_padding: (optional) An `ObservationPadding` enum value
        specifying the padding behavior of the initial buffers for delayed
        observables. If `ZERO` then the buffer is initially filled with zeroes.
        If `INITIAL_VALUE` then the buffer is initially filled with the first
        observation values.
      legacy_step: If True, steps the state with up-to-date position and
        velocity dependent fields. See Page 6 of
        https://arxiv.org/abs/2006.12983 for more information.
    """
    if not isinstance(delayed_observation_padding, ObservationPadding):
      raise ValueError(
          f'`delayed_observation_padding` should be an `ObservationPadding` '
          f'enum value: got {delayed_observation_padding}')

    self._task = task
    if not isinstance(random_state, np.random.RandomState):
      self._random_state = np.random.RandomState(random_state)
    else:
      self._random_state = random_state
    self._hooks = _EnvironmentHooks(self._task)
    self._time_limit = time_limit
    self._raise_exception_on_physics_error = raise_exception_on_physics_error
    self._strip_singleton_obs_buffer_dim = strip_singleton_obs_buffer_dim
    self._delayed_observation_padding = delayed_observation_padding
    self._legacy_step = legacy_step

    if n_sub_steps is not None:
      warnings.simplefilter('once', DeprecationWarning)
      warnings.warn('The `n_sub_steps` argument is deprecated. Please override '
                    'the `control_timestep` property of the task instead.',
                    DeprecationWarning)
    self._overridden_n_sub_steps = n_sub_steps

    self._recompile_physics_and_update_observables()

  def add_extra_hook(self, hook_name, hook_callable):
    self._hooks.add_extra_hook(hook_name, hook_callable)

  def _recompile_physics_and_update_observables(self):
    """Sets up the environment for latest MJCF model from the task."""
    self._physics_proxy = None
    self._recompile_physics()
    if isinstance(self._physics, weakref.ProxyType):
      self._physics_proxy = self._physics
    else:
      self._physics_proxy = weakref.proxy(self._physics)

    if self._overridden_n_sub_steps is not None:
      self._n_sub_steps = self._overridden_n_sub_steps
    else:
      self._n_sub_steps = self._task.physics_steps_per_control_step

    self._hooks.refresh_entity_hooks()
    self._hooks.after_compile(self._physics_proxy, self._random_state)
    self._observation_updater = self._make_observation_updater()
    self._observation_updater.reset(self._physics_proxy, self._random_state)

  def _recompile_physics(self):
    """Creates a new Physics using the latest MJCF model from the task."""
    if getattr(self, '_physics', None):
      self._physics.free()
    self._physics = mjcf.Physics.from_mjcf_model(
        self._task.root_entity.mjcf_model)
    self._physics.legacy_step = self._legacy_step

  def _make_observation_updater(self):
    pad_with_initial_value = (
        self._delayed_observation_padding == ObservationPadding.INITIAL_VALUE)
    return observation.Updater(
        self._task.observables, self._task.physics_steps_per_control_step,
        self._strip_singleton_obs_buffer_dim, pad_with_initial_value)

  @property
  def physics(self):
    """Returns a `weakref.ProxyType` pointing to the current `mjcf.Physics`.

    Note that the underlying `mjcf.Physics` will be destroyed whenever the MJCF
    model is recompiled. It is therefore unsafe for external objects to hold a
    reference to `environment.physics`. Attempting to access attributes of a
    dead `Physics` instance will result in a `ReferenceError`.
    """
    return self._physics_proxy

  @property
  def task(self):
    return self._task

  @property
  def random_state(self):
    return self._random_state

  def control_timestep(self):
    """Returns the interval between agent actions in seconds."""
    if self._overridden_n_sub_steps is not None:
      return self.physics.timestep() * self._overridden_n_sub_steps
    else:
      return self.task.control_timestep

class Environment(_CommonEnvironment, dm_env.Environment):
  """Reinforcement learning environment for Composer tasks."""

  def __init__(self, task, time_limit=float('inf'), random_state=None,
               n_sub_steps=None,
               raise_exception_on_physics_error=True,
               strip_singleton_obs_buffer_dim=False,
               max_reset_attempts=1,
               delayed_observation_padding=ObservationPadding.ZERO,
               legacy_step: bool = True):
    """Initializes an instance of `Environment`.

    Args:
      task: Instance of `composer.base.Task`.
      time_limit: (optional) A float, the time limit in seconds beyond which
        an episode is forced to terminate.
      random_state: (optional) an int seed or `np.random.RandomState` instance.
      n_sub_steps: (DEPRECATED) An integer, number of physics steps to take per
        agent control step. New code should instead override the
        `control_substep` property of the task.
      raise_exception_on_physics_error: (optional) A boolean, indicating whether
        `PhysicsError` should be raised as an exception. If `False`, physics
        errors will result in the current episode being terminated with a
        warning logged, and a new episode started.
      strip_singleton_obs_buffer_dim: (optional) A boolean, if `True`,
        the array shape of observations with `buffer_size == 1` will not have a
        leading buffer dimension.
      max_reset_attempts: (optional) Maximum number of times to try resetting
        the environment. If an `EpisodeInitializationError` is raised
        during this process, an environment reset is reattempted up to this
        number of times. If this count is exceeded then the most recent
        exception will be allowed to propagate. Defaults to 1, i.e. no failure
        is allowed.
      delayed_observation_padding: (optional) An `ObservationPadding` enum value
        specifying the padding behavior of the initial buffers for delayed
        observables. If `ZERO` then the buffer is initially filled with zeroes.
        If `INITIAL_VALUE` then the buffer is initially filled with the first
        observation values.
      legacy_step: If True, steps the state with up-to-date position and
        velocity dependent fields.
    """
    super().__init__(
        task=task,
        time_limit=time_limit,
        random_state=random_state,
        n_sub_steps=n_sub_steps,
        raise_exception_on_physics_error=raise_exception_on_physics_error,
        strip_singleton_obs_buffer_dim=strip_singleton_obs_buffer_dim,
        delayed_observation_padding=delayed_observation_padding,
        legacy_step=legacy_step)
    self._max_reset_attempts = max_reset_attempts
    self._reset_next_step = True

  def reset(self):
    failed_attempts = 0
    while True:
      try:
        return self._reset_attempt()
      except EpisodeInitializationError as e:
        failed_attempts += 1
        if failed_attempts < self._max_reset_attempts:
          logging.error('Error during episode reset: %s', repr(e))
        else:
          raise

  def _reset_attempt(self):
    self._hooks.initialize_episode_mjcf(self._random_state)
    self._recompile_physics_and_update_observables()
    with self._physics.reset_context():
      self._hooks.initialize_episode(self._physics_proxy, self._random_state)
    self._observation_updater.reset(self._physics_proxy, self._random_state)
    self._reset_next_step = False
    return dm_env.TimeStep(
        step_type=dm_env.StepType.FIRST,
        reward=None,
        discount=None,
        observation=self._observation_updater.get_observation())

  # TODO(b/129061424): Remove this method.
  def step_spec(self):
    """DEPRECATED: please use `reward_spec` and `discount_spec` instead."""
    warnings.warn('`step_spec` is deprecated, please use `reward_spec` and '
                  '`discount_spec` instead.', DeprecationWarning)
    if (self._task.get_reward_spec() is None or
        self._task.get_discount_spec() is None):
      raise NotImplementedError
    return dm_env.TimeStep(
        step_type=None,
        reward=self._task.get_reward_spec(),
        discount=self._task.get_discount_spec(),
        observation=self._observation_updater.observation_spec(),
    )

  def step(self, action):
    """Updates the environment using the action and returns a `TimeStep`."""
    if self._reset_next_step:
      self._reset_next_step = False
      return self.reset()

    self._hooks.before_step(self._physics_proxy, action, self._random_state)
    self._observation_updater.prepare_for_next_control_step()

    try:
      for i in range(self._n_sub_steps):
        self._substep(action)
        # The final observation update must happen after all the hooks in
        # `self._hooks.after_step` is called. Otherwise, if any of these hooks
        # modify the physics state then we might capture an observation that is
        # inconsistent with the final physics state.
        if i < self._n_sub_steps - 1:
          self._observation_updater.update()
      physics_is_divergent = False
    except control.PhysicsError as e:
      if not self._raise_exception_on_physics_error:
        logging.warning(e)
        physics_is_divergent = True
      else:
        raise

    self._hooks.after_step(self._physics_proxy, self._random_state)
    self._observation_updater.update()

    if not physics_is_divergent:
      reward = self._task.get_reward(self._physics_proxy)
      discount = self._task.get_discount(self._physics_proxy)
      terminating = (
          self._task.should_terminate_episode(self._physics_proxy)
          or self._physics.time() >= self._time_limit
      )
    else:
      reward = 0.0
      discount = 0.0
      terminating = True

    obs = self._observation_updater.get_observation()

    if not terminating:
      return dm_env.TimeStep(dm_env.StepType.MID, reward, discount, obs)
    else:
      self._reset_next_step = True
      return dm_env.TimeStep(dm_env.StepType.LAST, reward, discount, obs)

  def _substep(self, action):
    self._hooks.before_substep(
        self._physics_proxy, action, self._random_state)
    self._physics.step()
    self._hooks.after_substep(self._physics_proxy, self._random_state)

  def action_spec(self):
    """Returns the action specification for this environment."""
    return self._task.action_spec(self._physics_proxy)

  def reward_spec(self):
    """Describes the reward returned by this environment.

    This will be the output of `self.task.reward_spec()` if it is not None,
    otherwise it will be the default spec returned by
    `dm_env.Environment.reward_spec()`.

    Returns:
      A `specs.Array` instance, or a nested dict, list or tuple of
      `specs.Array`s.
    """
    task_reward_spec = self._task.get_reward_spec()
    if task_reward_spec is not None:
      return task_reward_spec
    else:
      return super().reward_spec()

  def discount_spec(self):
    """Describes the discount returned by this environment.

    This will be the output of `self.task.discount_spec()` if it is not None,
    otherwise it will be the default spec returned by
    `dm_env.Environment.discount_spec()`.

    Returns:
      A `specs.Array` instance, or a nested dict, list or tuple of
      `specs.Array`s.
    """
    task_discount_spec = self._task.get_discount_spec()
    if task_discount_spec is not None:
      return task_discount_spec
    else:
      return super().discount_spec()

  def observation_spec(self):
    """Returns the observation specification for this environment.

    Returns:
      An `OrderedDict` mapping observation name to `specs.Array` containing
      observation shape and dtype.
    """
    return self._observation_updater.observation_spec()


import os
import time
import warnings
import argparse

from IPython import embed

import numpy

from matplotlib import pyplot, ticker, colors

from ..etc import source, efficiency, telescopes, spectrum, extract, aperture, detector
from ..etc.observe import Observation
from . import scriptbase
from .. import data_file

def get_wavelength_vector(start, end, logstep):
    """
    Get the wavelength vector
    """
    nwave = int((numpy.log10(end)-numpy.log10(start))/logstep + 1)
    return numpy.power(10., numpy.arange(nwave)*logstep + numpy.log10(start))

def get_spectrum(wave, mag, emline_db=None, redshift=0.0, resolution=3500):
    """
    """
    spec = spectrum.ABReferenceSpectrum(wave, resolution=resolution, log=True)
    g = efficiency.FilterResponse()
    spec.rescale_magnitude(mag, band=g)
    if emline_db is None:
        return spec
    spec = spectrum.EmissionLineSpectrum(wave, emline_db['flux'], emline_db['restwave'],
                                         emline_db['fwhm'], units=emline_db['fwhmu'],
                                         redshift=redshift, resolution=resolution, log=True,
                                         continuum=spec.flux)
    warnings.warn('Including emission lines, spectrum g-band magnitude changed '
                  'from {0} to {1}.'.format(mag, spec.magnitude(band=g)))
    return spec

class FOBOSETCGrid(scriptbase.ScriptBase):

    @classmethod
    def get_parser(cls, width=None):
        import argparse

        parser = super().get_parser(description='Grid sampling of FOBOS ETC', width=width)

        parser.add_argument('-w', '--wavelengths', default=[3100,10000,4e-5], nargs=3, type=float,
                            help='Wavelength grid: start wave, approx end wave, logarithmic step.')
        parser.add_argument('-m', '--mag', default=[21.,25.,0.5], nargs=3, type=float,
                            help='Object apparent AB magnitude; assumed spectrum has a constant AB '
                                'otherwise, it is assumed to be the total magnitude of the source.')
        parser.add_argument('-t', '--time', default=[300.,3600.,300.], nargs=3, type=float,
                            help='Exposure time (s)')
        parser.add_argument('-f', '--fwhm', default=0.65, type=float,
                            help='On-sky PSF FWHM (arcsec)')
        parser.add_argument('-a', '--airmass', default=1.0, type=float, help='Airmass')
        parser.add_argument('--snr_units', type=str, default='pixel',
                            help='The units for the S/N.  Options are pixel, angstrom, resolution.')

        return parser

    @staticmethod
    def main(args):

        t = time.perf_counter()

        # Constants:
        resolution = 3500.      # lambda/dlambda
        fiber_diameter = 0.8    # Arcsec
        rn = 2.                             # Detector readnoise (e-)
        dark = 0.0                          # Detector dark-current (e-/s)

        # Temporary numbers that assume a given spectrograph PSF and LSF.
        # Assume 3 pixels per spectral and spatial FWHM.
        spatial_fwhm = 3.0
        spectral_fwhm = 3.0

        mags = numpy.arange(args.mag[0], args.mag[1]+args.mag[2], args.mag[2])
        times = numpy.arange(args.time[0], args.time[1]+args.time[2], args.time[2])

        # Get source spectrum in 1e-17 erg/s/cm^2/angstrom. Currently, the
        # source spectrum is assumed to be
        #   - normalized by the total integral of the source flux
        #   - independent of position within the source
        wave = get_wavelength_vector(args.wavelengths[0], args.wavelengths[1], args.wavelengths[2])
        spec = get_spectrum(wave, mags[0], resolution=resolution)

        # Get the source distribution.  If the source is uniform, onsky is None.
        onsky = None

        # Get the sky spectrum
        sky_spectrum = spectrum.MaunakeaSkySpectrum()

        # Overplot the source and sky spectrum
    #    ax = spec.plot()
    #    ax = sky_spectrum.plot(ax=ax, show=True)

        # Get the atmospheric throughput
        atmospheric_throughput = efficiency.AtmosphericThroughput(airmass=args.airmass)

        # Set the telescope. Defines the aperture area and throughput
        # (nominally 3 aluminum reflections for Keck)
        telescope = telescopes.KeckTelescope()

        # Define the observing aperture; fiber diameter is in arcseconds,
        # center is 0,0 to put the fiber on the target center. "resolution"
        # sets the resolution of the fiber rendering; it has nothing to do
        # with spatial or spectral resolution of the instrument
        fiber = aperture.FiberAperture(0, 0, fiber_diameter, resolution=100)

        # Get the spectrograph throughput (circa June 2018; TODO: needs to
        # be updated). Includes fibers + foreoptics + FRD + spectrograph +
        # detector QE (not sure about ADC). Because this is the total
        # throughput, define a generic efficiency object.
        thru_file = str(data_file(filename='efficiency') / 'fobos_throughput.db')
        thru_db = numpy.genfromtxt(thru_file)
        spectrograph_throughput = efficiency.Efficiency(thru_db[:,1], wave=thru_db[:,0])

        # System efficiency combines the spectrograph and the telescope
        system_throughput = efficiency.SystemThroughput(wave=spec.wave,
                                                        spectrograph=spectrograph_throughput,
                                                        telescope=telescope.throughput)

        # Instantiate the detector; really just a container for the rn and
        # dark current for now. QE is included in fobos_throughput.db file,
        # so I set it to 1 here.
        det = detector.Detector(rn=rn, dark=dark, qe=1.0)

        # Extraction: makes simple assumptions about the detector PSF for
        # each fiber spectrum and mimics a "perfect" extraction, including
        # an assumption of no cross-talk between fibers. Ignore the
        # "spectral extraction".
        extraction = extract.Extraction(det, spatial_fwhm=spatial_fwhm,
                                        spatial_width=1.5*spatial_fwhm,
                                        spectral_fwhm=spectral_fwhm, spectral_width=spectral_fwhm)

        snr_label = 'S/N per {0}'.format('R element' if args.snr_units == 'resolution'
                                        else args.snr_units)

        # SNR
        g = efficiency.FilterResponse()
        r = efficiency.FilterResponse(band='r')
        snr_g = numpy.empty((mags.size, times.size), dtype=float)
        snr_r = numpy.empty((mags.size, times.size), dtype=float)
        for i in range(mags.size):
            spec.rescale_magnitude(mags[i], band=g)
            for j in range(times.size):
                print('{0}/{1} ; {2}/{3}'.format(i+1,mags.size,j+1,times.size), end='\r')
                # Perform the observation
                obs = Observation(telescope, sky_spectrum, fiber, times[j], det,
                                system_throughput=system_throughput,
                                atmospheric_throughput=atmospheric_throughput,
                                airmass=args.airmass, onsky_source_distribution=onsky,
                                source_spectrum=spec, extraction=extraction,
                                snr_units=args.snr_units)

                # Construct the S/N spectrum
                snr_spec = obs.snr(sky_sub=True)
                snr_g[i,j] = numpy.sum(g(snr_spec.wave)*snr_spec.flux)/numpy.sum(g(snr_spec.wave))
                snr_r[i,j] = numpy.sum(r(snr_spec.wave)*snr_spec.flux)/numpy.sum(r(snr_spec.wave))
        print('{0}/{1} ; {2}/{3}'.format(i+1,mags.size,j+1,times.size))

        extent = [args.time[0] - args.time[2]/2, args.time[1] + args.time[2]/2,
                args.mag[0] - args.mag[2]/2, args.mag[1] + args.mag[2]/2]

        w,h = pyplot.figaspect(1)
        fig = pyplot.figure(figsize=(1.5*w,1.5*h))
        ax = fig.add_axes([0.15, 0.2, 0.7, 0.7])
        img = ax.imshow(snr_g, origin='lower', interpolation='nearest', extent=extent,
                        aspect='auto', norm=colors.LogNorm(vmin=snr_g.min(), vmax=snr_g.max()))
        cax = fig.add_axes([0.86, 0.2, 0.02, 0.7])
        pyplot.colorbar(img, cax=cax)
        cax.text(4, 0.5, snr_label, ha='center', va='center', transform=cax.transAxes,
                rotation='vertical')
        ax.text(0.5, -0.08, 'Exposure Time [s]', ha='center', va='center', transform=ax.transAxes)
        ax.text(-0.12, 0.5, r'Surface Brightness [AB mag/arcsec$^2$]', ha='center', va='center',
                transform=ax.transAxes, rotation='vertical')
        ax.text(0.5, 1.03, r'$g$-band S/N', ha='center', va='center', transform=ax.transAxes,
                fontsize=12)
        pyplot.show()


from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import absolute_import

import collections
import re

from . import _common as common
from ..base.candidate import CandidateFactory
from ..base.sentence import SentenceFactory
from ..base.mweoccur import MWEOccurrence
from ..base.word import Word
from .. import util

class TPCInfo(common.FiletypeInfo):
    r"""FiletypeInfo subclass for TaggedPlainCorpus format."""
    description = "One sentence per line with <mwepart>tags</mwepart>"
    filetype_ext = "TaggedPlainCorpus"

    comment_prefix = "#"
    escape_pairs = [("$", "${dollar}"), ("#", "${hash}"),
                    ("<", "${lt}"), (">", "${gt}"), (" ", "${space}"),
                    ("\n", "${newline}")]

    def operations(self):
        return common.FiletypeOperations(TPCChecker, TPCParser, TPCPrinter)

class TPCChecker(common.AbstractChecker):
    r"""Checks whether input is in TaggedPlainCorpus format."""
    def matches_header(self, strict):
        return not strict or "<mwepart" in self.fileobj.peek(1024)

class TPCParser(common.AbstractTxtParser):
    r"""Instances of this class parse the TaggedPlainCorpus format,
    calling the `handler` for each object that is parsed.
    """
    valid_categories = ["corpus"]
    RE_ENTRY = re.compile(
            "(?P<word>[^<> ]+)" \
            "|(?P<complex><mwepart +id=\"(?P<ids>[0-9,]*)\">" \
                    "(?P<c_word>[^<> ]+)</ *mwepart *>)")

    def __init__(self, encoding='utf-8'):
        super(TPCParser, self).__init__(encoding)
        self.candidate_factory = CandidateFactory()
        self.sentence_factory = SentenceFactory()
        self.category = "corpus"

    def _parse_line(self, line, info={}):
        sentence = self.sentence_factory.make()
        num2cand = collections.defaultdict(list)
        num2indexes = collections.defaultdict(list)

        for match in self.RE_ENTRY.finditer(line):
            word, complex, ids, c_word = match.groups()
            if word:
                sentence.append(Word(word))
            else:
                sentence.append(Word(c_word))
                for id in ids.split(","):
                    num2cand[int(id)].append(sentence[-1])
                    num2indexes[int(id)].append(len(sentence)-1)

        for num in sorted(num2cand.keys()):
            words = num2cand[ num ]
            c = self.candidate_factory.make(words, id_number=num)
            mweo = MWEOccurrence(sentence, c, num2indexes[num])
            sentence.mweoccurs.append(mweo)

        self.handler.handle_sentence(sentence, info)

class TPCPrinter(common.AbstractPrinter):
    """Instances can be used to print HTML format."""
    valid_categories = ["corpus"]

    def handle_sentence(self, sentence, info={}):
        """Print a simple readable string where the surface forms of the
        current sentence are concatenated and separated by a single space.

        @return A string with the surface form of the sentence,
        space-separated.
        """
        surface_list = [self.escape(w.surface) for w in sentence.word_list]
        mwetags_list = [[] for i in range(len(surface_list))]
        for mweoccur in sentence.mweoccurs:
            for i in mweoccur.indexes:
                mwetags_list[i].append( mweoccur.candidate.id_number)
        for (mwetag_i, mwetag) in enumerate(mwetags_list):
            if mwetag:
                mwetag = (unicode(index) for index in mwetag)
                surface_list[mwetag_i] = "<mwepart id=\"" + ",".join(mwetag) \
                              + "\">" + surface_list[mwetag_i] + "</mwepart>"
        line = " ".join(surface_list)
        self.add_string(line, "\n")


"""
File name: cache.py
Description: Home page API route

Change Log:
Release Date    Revision Date   Changes By      Description
------------    -------------   ----------      -----------
                July 2020       Sabarish AC     Initial
                August 2020     Ganapathy R     Code Updation & Refraction
"""
# Built in Modules
from flask import render_template, abort, session, flash, redirect, url_for

# Customized Modules
from . import home
import sys
sys.path.append('C:\\ctpt\\ctpt-backup\\resource_integrate')
from util import util

conf = util.get_conf()
const = util.excelConstants()

"""
Render to Home Page on / route
"""
@home.route('/home')
@home.route('/')
def homePage():
    if 'email' in session:
        email = session['email']
        return render_template(conf['templates']['home'], logged_in=session, is_home=True, title='Welcome to CTPT Shift Plan App')
    else:
        flash('You muste be logged in to use this app, please login!!', category='warning')
        return redirect(url_for('auth.user_login'))


# -*- coding: utf-8 -*-
"""
Created on Thu Oct 13 10:29:07 2016

@author: d_floriello

Outages analysis
"""

import pandas as pd
import datetime
import numpy as np
from collections import OrderedDict
import matplotlib.pyplot as plt

out = pd.read_excel('C:/Users/d_floriello/Documents/ourages_fran_2016-10-12.xlsx')

today = datetime.datetime(2016, 10, 24)

out = out.ix[out['Fin Indispo'] >= today]

###############################################################################
def update_outages(out, today):
    out = out.ix[out['Fin Indispo'] >= today]
    ids = np.unique(out['ID Indisponibilité de production'])
    ixl = []
    for ii in ids:
        given = out.ix[out['ID Indisponibilité de production'] == ii]
        ixl.append(max(given.index.tolist()))
    out = out[['ID Indisponibilité de production', "Type d'indisponibilité", 'Filière',
       "Type de l'unité de production",  'Nom du producteur',
       'Puissance nominale', 'Puissance disponible restante', 'Cause',
       'Statut']]
    return out.ix[ixl]
###############################################################################
def get_statistics(out, by, varname):
    diz1 = OrderedDict()
    diz2 = OrderedDict()
#    categorical = ["Type d'indisponibilité", 'Filière',
#       "Type de l'unité de production",  'Nom du producteur','Cause',
#       'Statut']
    grouped = out.groupby(by = by)
    names = list(set(out[by].tolist()))
#    if varname in categorical:
    for i in names:
        il1 = []
        il2 = []
        for vn in list(set(out[varname].tolist())):
            il1.append(np.where(grouped.get_group(i)[varname] == vn)[0].size)
        il2 = [i/sum(il1) if sum(il1) > 0 else 0 for i in il1]
        diz1[i] = il1
        diz2[i] = il2
    df1 = pd.DataFrame.from_dict(diz1, orient = 'index')
    df2 = pd.DataFrame.from_dict(diz2, orient = 'index')
    df1.columns = [list(set(out[varname].tolist()))]
    df2.columns = [list(set(out[varname].tolist()))]

    return df1, df2
###############################################################################
def get_statistics_all(out):
    varnames = ["Type d'indisponibilité", 'Filière',
       "Type de l'unité de production",  'Nom du producteur',
       'Puissance nominale', 'Puissance disponible restante', 'Cause']
#    categorical = ["Type d'indisponibilité", 'Filière',
#       "Type de l'unité de production",  'Nom du producteur','Cause',
#       'Statut']
    for vn in varnames:
        remains = set(varnames).difference(vn)
        for rmn in remains:
            S, P = get_statistics(out, vn, rmn)
            print('Statistics of {} vs {}:'.format(vn, rmn))
            print(S)
            print(P)
            print('##################################')
###############################################################################
def get_simple_count(out, month, var):
    today = datetime.datetime(2016, month, 1)
    out2 = update_outages(out, today)
    print('num outages totali nel mese {}= {}'.format(month, out2.shape[0]))
    vn = set(list(out2[var]))
    cts = []
    for v in vn:
        print('percentage of {} outages = {}'.format(v, out2[var].ix[out2[var] == v].count()/out2.shape[0]))
        cts.append(out2[var].ix[out2[var] == v].count()/out2.shape[0])
    return cts
###############################################################################
def get_remaining_power(out, month):
    today = datetime.datetime(2016, month, 1)
    out2 = update_outages(out, today)
    out2 = out2.reset_index(drop = True)
    print('num outages totali nel mese {}= {}'.format(month, out2.shape[0]))
    remaining = perc = 0
    for r in range(out2.shape[0]):
        remaining += out2['Puissance nominale'].ix[r] - out2['Puissance disponible restante'].ix[r]
    perc = remaining/out2['Puissance nominale'].sum()
    return remaining, perc
###############################################################################
out2 = update_outages(out, today)

varnames = ['ID Indisponibilité de production', "Type d'indisponibilité", 'Filière',
       "Type de l'unité de production",  'Nom du producteur',
       'Puissance nominale', 'Puissance disponible restante', 'Cause',
       'Statut']

S, P = get_statistics(out2, varnames[2], varnames[6])
P.sum(axis = 1)

###############################################################################
##################################### 2015 ####################################
###############################################################################

out = pd.read_excel('C:/Users/d_floriello/Documents/out2015.xlsx')

nmn = ['num outages tot', 'tot indisp', '%tot indisp', 'altro_filiera', 'Hydraulique lacs',
 "Hydraulique fil de l'eau / éclusée",
 'Nucléaire',
 'Charbon',
 'Autre',
 'Gaz',
 'Marin',
 'Fioul',
 'Hydraulique STEP',
 'altro',
 'Indisponibilité planifiée', 'Indisponibilité fortuite']#,
# 'altro_prod',
# 'DIRECT ENERGIE',
# 'TOTAL',
# 'GDF-SUEZ',
# 'UNIPER',
# 'EDF',
# 'PSS POWER',
# 'ALPIQ']

diz = OrderedDict()
# il DataFrame risultante verra riempito da diz per righe.
# ogni entrata di diz sarà:
# 0: num outages totali;
# 1: indisponibilita totale in MWh(?)
# 2: indisponibilita inpercentuale
# 3: per ogni tipo di centrale la percentuale fuori uso di quel tipo sul totale outages
# 4: percentuale cause
# 5: percentuale produttori
mon = [1,2,3,4,5,6,7,8,9,10,11,12]
filiera = list(set(out['Filière']))
tipo = list(set(out["Type d'indisponibilité"]))
prod = list(set(out['Nom du producteur']))
cause = list(set(out['Cause']))
filiera = [i for i in filiera if isinstance(i, str)]
tipo = [i for i in tipo if isinstance(i, str)]
prod = [i for i in prod if isinstance(i, str)]
cause = [i for i in cause if isinstance(i, str)]
for m in mon:
    vec = []
    out2 = out.set_index(out['Fin Indispo'])
    atm = out2.ix[out2.index.month > m]
    ids = list(set(atm['ID Indisponibilité de production']))
    ind = tot = 0
    nfil = np.repeat(0, len(filiera)).tolist()
    ntip = np.repeat(0, len(tipo)).tolist()
    npro = np.repeat(0, len(prod)).tolist()
    ncau = np.repeat(0, len(cause)).tolist()
    if len(ids) > 0:
        vec.append([len(ids)]) ### num outages totali
        for i in ids:
#            print(ind)
#            print(type(ind))
#            print(tot)
#            print(type(tot))
            I = atm.ix[atm['ID Indisponibilité de production'] == i]
            M = I.ix[I['Version'] == np.max(I['Version'])]
            ind += M['Puissance nominale'].values[0] - M['Puissance disponible restante'].values[0]
            tot += M['Puissance nominale'].values[0]
#            print(ind)
#            print(type(ind))
#            print(tot)
#            print(type(tot))
            for fil in filiera:
                if fil == M['Filière'].values[0]:
                    indx = filiera.index(fil)
                    nfil[indx] += 1
            for tip in tipo:
                if tip == M["Type d'indisponibilité"].values[0]:
                   #print(M["Type d'indisponibilité"].values[0])
                   indx = tipo.index(tip)
                   ntip[indx] += 1
                   #print(ntip)
            for pro in prod:
                if pro == M['Nom du producteur'].values[0]:
                    indx = prod.index(pro)
                    npro[indx] += 1
            for cu in cause:
                if cu == M['Cause'].values[0]:
                    indx = cause.index(cu)
                    ncau[indx] += 1
        perc = ind/tot
        print(perc)
        vec.append([ind])
        vec.append([perc])
        print((np.array(ntip)/len(ids)).tolist())
        vec.append((np.array(nfil)/len(ids)).tolist())
        #vec.append((np.array(ntip)/len(ids)).tolist())
        vec.append([ntip[0]/len(ids)])
        vec.append([ntip[1]/len(ids)])
        #vec.append((np.array(npro)/len(ids)).tolist())
        #vec.append(ncau/ids.size)
        vec= [item for sublist in vec for item in sublist]
        #vec = np.array(vec).ravel().tolist()
        diz[str(m)] = vec
    else:
        diz[str(m)] = np.repeat(0,len(nmn)).tolist()

DF = pd.DataFrame.from_dict(diz, orient = 'index')
DF.columns = [nmn]
DF.shape

fs = pd.read_excel('C:/Users/d_floriello/Documents/Prezzi Francia e Svizzera (2015 -2016).xlsx', sheetname = '2015')
fs = fs[fs.columns[[2,3]]].set_index(fs['Data'])
plt.figure()
plt.plot(fs[fs.columns[0]].resample('D').mean())
plt.figure()
DF[DF.columns[3:13]].plot()

from pandas.tools import plotting

plt.figure()
plotting.parallel_coordinates(DF[DF.columns[3:13]])

for i in range(1,13,1):
    print(sorted(DF[DF.columns[3:13]].ix[DF.index == str(i)]))

nuc = []
hydro = []
gas = []
coal = []
rim = []
perc = []
for i in range(1,13,1):
    nuc.append(get_simple_count(out, i, 'Filière')[3])
    hydro.append(get_simple_count(out, i, 'Filière')[2] + get_simple_count(out, i, 'Filière')[5])
    gas.append(get_simple_count(out, i, 'Filière')[1])
    coal.append(get_simple_count(out, i, 'Filière')[0])
    rim.append(get_remaining_power(out, i)[0])
    perc.append(get_remaining_power(out, i)[1])

diz2 = OrderedDict()
diz2['nucleare'] = nuc
diz2['hydro'] = hydro
diz2['gas'] = gas
diz2['carbone'] = coal

diz3 = OrderedDict()
diz3['remaining power'] = rim
diz3['%remaining power'] = perc

D2 = pd.DataFrame.from_dict(diz2)
D3 = pd.DataFrame.from_dict(diz3)

plt.figure()
plt.plot(np.array(nuc))
plt.figure()
plt.plot(np.array(fs[fs.columns[0]].resample('M').mean()))
plt.figure()
D3['%remaining power'].plot()

plt.figure()
plt.scatter(np.array(D3['remaining power']), np.array(fs[fs.columns[0]].resample('M').mean()))


#!/Users/dmitrychernyshev/Projects/stepik_python/venv/bin/python3
import json
import sys
import requests

params = {
	'default': 'boring',
	'fragment': ''
	}

for line in sys.stdin:
	line = line.strip()
	url = "http://numbersapi.com/" + line + "/math"

	res = requests.get(url, params=params)
	print('Interesting' if res.text != 'boring' else 'Boring')


# -*- coding: utf-8 -*-
from Products.CMFCore.utils import getToolByName
from Products.Five.browser import BrowserView
from Products.Five.browser.pagetemplatefile import ViewPageTemplateFile

from bs4 import BeautifulSoup
from plone.app.contenttypes.interfaces import IFolder
from plone.dexterity.content import Container
from pyquery import PyQuery as pq
from requests.exceptions import ReadTimeout
from requests.exceptions import RequestException
from zope import schema
from zope.annotation.interfaces import IAnnotations
from zope.component import getAdapter, getAdapters
from zope.interface import implementer
from zope.interface import Interface

from genweb6.core import utils
from genweb6.upc import _
from genweb6.upc.content.packet import PACKETS_KEY

import plone.api
import re
import requests
import urllib.parse

class Ipacket(IFolder):
    """Marker interface for the packet"""

@implementer(Ipacket)
class Packet(Container):

    @property
    def b_icon_expr(self):
        return "box"

class IpacketDefinition(Interface):
    """A packet definition"""

    order = schema.Int(title=u"Order")
    URL_schema = schema.TextLine(title=u"The URL schema of the packet")

class View(BrowserView):

    template = ViewPageTemplateFile('templates/view.pt')

    def __init__(self, context, request):
        self.context = context
        self.request = request
        self.title = ''
        self.content = ''

    def __call__(self):
        return self.template()

    def get_catalog_content(self, path_to_search):
        """ Fem una consulta al catalog, en comptes de fer un PyQuery """
        raw_html = u''
        catalog = getToolByName(self.context, 'portal_catalog')

        objects = catalog(path=path_to_search)
        try:
            raw_html = objects[0]()
        except:
            raw_html = objects[0].getObject()()

        return raw_html

    def getHTML(self):
        packet_type = self.getType()
        adapter = getAdapter(self.context, IpacketDefinition, packet_type)
        adapter.packet_fields.update({'lang': utils.pref_lang()})

        url = adapter.URL_schema
        portal_url = getToolByName(self.context, "portal_url")
        portal = portal_url.getPortalObject()
        url_portal_nginx = portal.absolute_url()  # url (per dns) del lloc

        try:
            url = self.get_absolute_url(url % adapter.packet_fields)
            # check url to avoid autoreference, removing http(s) and final slash
            link_url = re.findall('https?://(.*)', url)[0].strip('/')  # url del contingut netejada
            parent_url = re.findall('https?://(.*)', self.context.absolute_url())[0].strip('/')  # url del pare netejada
            root_url = re.findall('https?://(.*)', url_portal_nginx)[0].strip('/')  # url (per dns) del lloc netejada
            if link_url != parent_url:
                if link_url.startswith(root_url):
                    # link intern, search through the catalog
                    relative_path = '/' + re.findall(root_url + '(.*)', link_url)[0]
                    url_to_search = '/'.join(portal.getPhysicalPath()) + relative_path
                    raw_html = self.get_catalog_content(url_to_search)
                    clean_html = re.sub(r'[\n\r]?', r'', raw_html)
                    doc = pq(clean_html)
                    match = re.search(r'This page does not exist', clean_html)
                    self.title = self.context.Title()  # titol per defecte
                    if not match:
                        if packet_type == 'contingut_genweb':
                            element = adapter.packet_fields['element']
                            if not element:
                                element = "#content-core"
                        else:
                            element = "#content-nucli"
                        content = pq('
').append(
                            doc(element).outerHtml()).html(method='html')
                        if not content:
                            content = _(u"ERROR. This element does not exist.") + " " + element
                    else:
                        content = _(u"ERROR: Unknown identifier. This page does not exist." + url)
                else:
                    # link extern, pyreq
                    raw_html = requests.get(url, timeout=5, verify=False)
                    clean_html = re.sub(r'[\n\r]?', r'', raw_html.text)
                    doc = pq(clean_html)
                    match = re.search(r'This page does not exist', clean_html)
                    self.title = self.context.Title()  # titol per defecte
                    if not match:
                        if packet_type == 'contingut_genweb':
                            element = adapter.packet_fields['element']
                            if not element:
                                element = "#content-core"
                        else:
                            element = "#content-nucli"
                        content = pq('
').append(
                            doc(element).outerHtml()).html(method='html')
                        if not content:
                            content = _(u"ERROR. This element does not exist.") + " " + element
                    else:
                        content = _(u"ERROR: Unknown identifier. This page does not exist." + url)
            else:
                content = _(u"ERROR. Autoreference")
        except ReadTimeout:
            content = _(u"ERROR. There was a timeout while waiting for '{0}'".format(self.get_absolute_url(self.data.url)))
        except RequestException:
            content = _(u"ERROR. This URL does not exist.")
        except:
            content = _(u"ERROR. Unexpected exception.")

        soup = BeautifulSoup(clean_html, "html.parser")
        body = soup.find_all("body")
        if body:
            class_body = body[0].get("class", [])
            valid_class = [valid for valid in class_body if valid.startswith('template-') or valid.startswith('portaltype-')]
            content = str('
' + content + '
')

        self.content = content

    def get_absolute_url(self, url):
        """
        Convert relative url to absolute
        """
        if not ("://" in url):
            base = self.context.__parent__.absolute_url() + '/'
            return urllib.parse.urljoin(base, url)
        else:
            # Already absolute
            return url

    def getPacket(self):
        return self.content

    def getTitle(self):
        # import pdb; pdb.set_trace()
        # si el contingut esta configurat i encara no tenim el content, l'obtenim
        # el titol sera el nom del contingut (llistat, paquet o genweb) o el nom de l'estudi
        if self.isAlreadyConfigured() and self.content == '':
            self.getHTML()
        return self.title

    def getType(self):
        annotations = IAnnotations(self.context)
        return annotations.get(PACKETS_KEY + '.type', None)

    def isAlreadyConfigured(self):
        annotations = IAnnotations(self.context)
        if annotations.get(PACKETS_KEY + '.type', None):
            return True
        else:
            return False

    def selectedPacket(self):
        annotations = IAnnotations(self.context)
        packet_key = annotations.get(PACKETS_KEY + '.type')
        packet_fields = annotations.get(PACKETS_KEY + '.fields')
        packet_mapui = annotations.get(PACKETS_KEY + '.mapui')

        return dict(packet_key=packet_key, value=packet_fields.get(packet_mapui.get('codi')), element=packet_fields.get(packet_mapui.get('element')))

    def show_extended_info(self):
        user = plone.api.user.get_current()

        if getattr(user, 'name', False):
            if user.name == 'Anonymous User':
                return False
        try:
            user_roles = set(plone.api.user.get_roles(user=user, obj=self.context) +
                             plone.api.user.get_roles(user=user))
        except:
            user_roles = set(plone.api.user.get_roles(user=user, obj=self.context))

        if 'Manager' in user_roles or \
           'WebMaster' in user_roles or \
           'Site Administrator' in user_roles or \
           'Owner' in user_roles:
            return True
        else:
            return False

    def absolute_url(self, url):
        """
        Convert relative url to absolute
        """
        if not ("://" in url):
            base = self.context.__parent__.absolute_url() + '/'
            return urllib.parse.urljoin(base, url)
        else:
            # Already absolute
            return url

class Configure(BrowserView):

    template = ViewPageTemplateFile('templates/configure.pt')

    def __init__(self, context, request):
        self.context = context
        self.request = request
        self.available_ptypes = sorted([adapter for adapter in getAdapters((self.context,), IpacketDefinition)], key=lambda adapter: adapter[1].order)

    def __call__(self):
        if self.request.form:
            form = self.request.form
            packet_type = form.get("packet_type", None)
            if packet_type:
                adapter = getAdapter(self.context, IpacketDefinition, packet_type)
                field_values = dict([(field, form[field]) for field in adapter.fields])
                adapter.packet_fields = field_values
                adapter.packet_type = packet_type
                adapter.packet_mapui = adapter.mapui
                return self.request.response.redirect(self.context.absolute_url())
        return self.template()

    def getAvailablePacketsInfo(self):
        return self.available_ptypes

    def getTitle(self):
        return self.context.Title()

    def selectedPacket(self):
        if self.isAlreadyConfigured():
            annotations = IAnnotations(self.context)
            packet_key = annotations.get(PACKETS_KEY + '.type')
            packet_fields = annotations.get(PACKETS_KEY + '.fields')
            packet_mapui = annotations.get(PACKETS_KEY + '.mapui')
            selected = dict(packet_key=packet_key, value=packet_fields.get(packet_mapui.get('codi')), element=packet_fields.get(packet_mapui.get('element')))
        else:
            selected = {'packet_key': '', 'value': '', 'element': ''}

        return selected

    def isAlreadyConfigured(self):
        annotations = IAnnotations(self.context)
        if annotations.get(PACKETS_KEY + '.type', None):
            return True
        else:
            return False

    def getType(self):
        annotations = IAnnotations(self.context)
        return annotations.get(PACKETS_KEY + '.type', None)


import os
from flask import Flask, jsonify, g
from flask_cors import CORS
from flask_login import LoginManager

import models
#import resources
from resources.dates import date
from resources.users import user
from resources.creates import create

DEBUG = True
PORT = 8000

# Initialize an instance of the Flask class.
app = Flask(__name__)

app.secret_key = "ladedaladeda" #secret key encodes session
login_manager = LoginManager()
login_manager.init_app(app) #sets up sessions

@login_manager.user_loader #loads user object when we access the session
#grab user by importing current_user from flask_login
def load_user(userid):
    try:
        return models.User.get(models.User.id == userid)
    except: models.DoesNotExist
    return None

@app.before_request
def before_request():
    """Connect to the database before each request."""
    g.db = models.DATABASE
    g.db.connect()

@app.after_request
def after_request(response):
    """Close the database connection after each request."""
    g.db.close()
    return response

# CORS(app, origins=['http://localhost:3000'], supports_credentials=True)
#supports_credentials = true means we allow cookies to be sent to the server
CORS(date, origins=['http://localhost:3000', 'https://react-find-a-date-idea.herokuapp.com'], supports_credentials=True)
app.register_blueprint(date,  url_prefix='/api/v1/dates')
CORS(user, origins=['http://localhost:3000', 'https://react-find-a-date-idea.herokuapp.com'], supports_credentials=True)
app.register_blueprint(user,  url_prefix='/api/v1/users')
CORS(create, origins=['http://localhost:3000', 'https://react-find-a-date-idea.herokuapp.com'], supports_credentials=True)
app.register_blueprint(create,  url_prefix='/api/v1/creates')

# Run the app
if 'ON_HEROKU' in os.environ:
    print('hitting')
    models.initialize()

if __name__ == '__main__':
    models.initialize()
    app.run(debug=DEBUG, port=PORT)


import pprint
import os
import sys

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from request import BaseRequest
from dotenv import dotenv_values

config = dotenv_values()

# Example of GET /invoices/:invoice_id

class GetInvoiceRequest(BaseRequest):
    def __init__(self):
        super().__init__()

    def get_invoice(self, id):
        path = f"/invoices/{id}"
        return self.make_request("GET", path)

# run as standalone script by passing any command line argument
if len(sys.argv) > 1:
    invoice_id = config['INVOICE_ID']

    get_invoice_request = GetInvoiceRequest()
    invoice = get_invoice_request.get_invoice(invoice_id )

    pprint.pprint(invoice)


from flask import Flask, render_template, redirect, url_for, session
from flask.ext.bootstrap import Bootstrap
from flask.ext.wtf import Form
from wtforms import StringField, \
    SelectField, RadioField, TextAreaField, SubmitField
from wtforms.validators import Required, Email
import logging
import sys

app = Flask(__name__)
app.config.from_pyfile('maprequests.cfg')
bootstrap = Bootstrap(app)
app.logger.addHandler(logging.StreamHandler(sys.stdout))
app.logger.setLevel(logging.ERROR)

class MapForm(Form):
    name = StringField('Name:', validators=[Required()])
    email = StringField('Email:', validators=[Required(), Email()])
    pageSize = SelectField(
        'Page Size:',
        choices=[('std', '8.5x11'), ('lg', '11x17')],
        validators=[Required()])
    orientation = RadioField(
        'Page Orientation:',
        choices=[('portrait', 'Portrait'), ('landscape', 'Landscape')],
        validators=[Required()])
    submit = SubmitField('Next')

@app.route('/', methods=['GET', 'POST'])
def index():
    form = MapForm()
    if form.validate_on_submit():
        session['name'] = form.name.data
        session['email'] = form.email.data
        session['pageSize'] = form.pageSize.data
        session['orientation'] = form.orientation.data
        return redirect(url_for('index'))
    return render_template('index.html', form=form)

if __name__ == '__main__':
    app.run()


"""ag.Space sanitizer for certain hyperparameters."""
import warnings

import numpy as np
from autogluon.core import Categorical, Int

def sanitize_batch_size(batch_size, min_value=1, max_value=np.inf):
    if isinstance(batch_size, Categorical):
        valid_bs = []
        bs_values = batch_size.data
        for bs_value in bs_values:
            if isinstance(bs_value, int) and min_value < bs_value < max_value:
                valid_bs.append(bs_value)
        if valid_bs != bs_values:
            warnings.warn(
                f'Pruning batch size from {batch_size} to {valid_bs} due to memory limit.'
            )
        if len(valid_bs) == 1:
            new_bs = valid_bs[0]
        else:
            new_bs = Categorical(*valid_bs)
    elif isinstance(batch_size, Int):
        lower = batch_size.lower
        upper = batch_size.upper
        if not isinstance(lower, int) or not isinstance(upper, int):
            raise TypeError(
                f'Invalid lower {lower} or upper {upper} bound for Int space')
        lower = max(lower, min_value)
        upper = min(upper, max_value)
        new_bs = Int(lower=lower, upper=upper)
        if lower != batch_size.lower or upper != batch_size.higher:
            warnings.warn(
                f'Adjusting batch size range from {batch_size} to {new_bs} due to memory limit.'
            )
    elif isinstance(batch_size, int):
        new_bs = max(min(batch_size, max_value), min_value)
        if new_bs != batch_size:
            warnings.warn(
                f'Adjusting batch size from {batch_size} to {new_bs} due to memory limit.'
            )
    else:
        raise TypeError(
            f'Expecting batch size to be (Categorical/Int/int), given {type(batch_size)}.'
        )
    return new_bs


#20220227
#백준(투포인터) : 21921 블로그#https://www.acmicpc.net/problem/21921

import sys
#누적 합
from itertools import accumulate
input = sys.stdin.readline

n, x = map(int,input().split())
s = 0
hap = []
a = list(map(int,input().split()))
b = list(accumulate(a))

for i in range(1,n):
  if (i-x) < 0:
    hap.append(b[i])
  else:
    hap.append(b[i]-b[i-x])

if max(hap) != 0:
  print(max(hap))
else:
  print('SAD')
  exit(0)

print(hap.count(max(hap)))