import argparse
import collections
import imblearn
import joblib
import json
import numpy as np
import pandas as pd
import pickle
import skrebate
import sklearn
import sys
import xgboost
import warnings
from imblearn import under_sampling, over_sampling, combine
from scipy.io import mmread
from mlxtend import classifier, regressor
from sklearn.base import clone
from sklearn import (cluster, compose, decomposition, ensemble,
                     feature_extraction, feature_selection,
                     gaussian_process, kernel_approximation, metrics,
                     model_selection, naive_bayes, neighbors,
                     pipeline, preprocessing, svm, linear_model,
                     tree, discriminant_analysis)
from sklearn.exceptions import FitFailedWarning
from sklearn.model_selection._validation import _score, cross_validate
from sklearn.model_selection import _search, _validation

from galaxy_ml.utils import (SafeEval, get_cv, get_scoring, load_model,
                             read_columns, try_get_attr, get_module)

_fit_and_score = try_get_attr('galaxy_ml.model_validations', '_fit_and_score')
setattr(_search, '_fit_and_score', _fit_and_score)
setattr(_validation, '_fit_and_score', _fit_and_score)

N_JOBS = int(__import__('os').environ.get('GALAXY_SLOTS', 1))
CACHE_DIR = './cached'
NON_SEARCHABLE = ('n_jobs', 'pre_dispatch', 'memory', '_path',
                  'nthread', 'callbacks')
ALLOWED_CALLBACKS = ('EarlyStopping', 'TerminateOnNaN', 'ReduceLROnPlateau',
                     'CSVLogger', 'None')

def _eval_search_params(params_builder):
    search_params = {}

    for p in params_builder['param_set']:
        search_list = p['sp_list'].strip()
        if search_list == '':
            continue

        param_name = p['sp_name']
        if param_name.lower().endswith(NON_SEARCHABLE):
            print("Warning: `%s` is not eligible for search and was "
                  "omitted!" % param_name)
            continue

        if not search_list.startswith(':'):
            safe_eval = SafeEval(load_scipy=True, load_numpy=True)
            ev = safe_eval(search_list)
            search_params[param_name] = ev
        else:
            # Have `:` before search list, asks for estimator evaluatio
            safe_eval_es = SafeEval(load_estimators=True)
            search_list = search_list[1:].strip()
            # TODO maybe add regular express check
            ev = safe_eval_es(search_list)
            preprocessings = (
                preprocessing.StandardScaler(), preprocessing.Binarizer(),
                preprocessing.MaxAbsScaler(),
                preprocessing.Normalizer(), preprocessing.MinMaxScaler(),
                preprocessing.PolynomialFeatures(),
                preprocessing.RobustScaler(), feature_selection.SelectKBest(),
                feature_selection.GenericUnivariateSelect(),
                feature_selection.SelectPercentile(),
                feature_selection.SelectFpr(), feature_selection.SelectFdr(),
                feature_selection.SelectFwe(),
                feature_selection.VarianceThreshold(),
                decomposition.FactorAnalysis(random_state=0),
                decomposition.FastICA(random_state=0),
                decomposition.IncrementalPCA(),
                decomposition.KernelPCA(random_state=0, n_jobs=N_JOBS),
                decomposition.LatentDirichletAllocation(
                    random_state=0, n_jobs=N_JOBS),
                decomposition.MiniBatchDictionaryLearning(
                    random_state=0, n_jobs=N_JOBS),
                decomposition.MiniBatchSparsePCA(
                    random_state=0, n_jobs=N_JOBS),
                decomposition.NMF(random_state=0),
                decomposition.PCA(random_state=0),
                decomposition.SparsePCA(random_state=0, n_jobs=N_JOBS),
                decomposition.TruncatedSVD(random_state=0),
                kernel_approximation.Nystroem(random_state=0),
                kernel_approximation.RBFSampler(random_state=0),
                kernel_approximation.AdditiveChi2Sampler(),
                kernel_approximation.SkewedChi2Sampler(random_state=0),
                cluster.FeatureAgglomeration(),
                skrebate.ReliefF(n_jobs=N_JOBS),
                skrebate.SURF(n_jobs=N_JOBS),
                skrebate.SURFstar(n_jobs=N_JOBS),
                skrebate.MultiSURF(n_jobs=N_JOBS),
                skrebate.MultiSURFstar(n_jobs=N_JOBS),
                imblearn.under_sampling.ClusterCentroids(
                    random_state=0, n_jobs=N_JOBS),
                imblearn.under_sampling.CondensedNearestNeighbour(
                    random_state=0, n_jobs=N_JOBS),
                imblearn.under_sampling.EditedNearestNeighbours(
                    random_state=0, n_jobs=N_JOBS),
                imblearn.under_sampling.RepeatedEditedNearestNeighbours(
                    random_state=0, n_jobs=N_JOBS),
                imblearn.under_sampling.AllKNN(random_state=0, n_jobs=N_JOBS),
                imblearn.under_sampling.InstanceHardnessThreshold(
                    random_state=0, n_jobs=N_JOBS),
                imblearn.under_sampling.NearMiss(
                    random_state=0, n_jobs=N_JOBS),
                imblearn.under_sampling.NeighbourhoodCleaningRule(
                    random_state=0, n_jobs=N_JOBS),
                imblearn.under_sampling.OneSidedSelection(
                    random_state=0, n_jobs=N_JOBS),
                imblearn.under_sampling.RandomUnderSampler(
                    random_state=0),
                imblearn.under_sampling.TomekLinks(
                    random_state=0, n_jobs=N_JOBS),
                imblearn.over_sampling.ADASYN(random_state=0, n_jobs=N_JOBS),
                imblearn.over_sampling.RandomOverSampler(random_state=0),
                imblearn.over_sampling.SMOTE(random_state=0, n_jobs=N_JOBS),
                imblearn.over_sampling.SVMSMOTE(random_state=0, n_jobs=N_JOBS),
                imblearn.over_sampling.BorderlineSMOTE(
                    random_state=0, n_jobs=N_JOBS),
                imblearn.over_sampling.SMOTENC(
                    categorical_features=[], random_state=0, n_jobs=N_JOBS),
                imblearn.combine.SMOTEENN(random_state=0),
                imblearn.combine.SMOTETomek(random_state=0))
            newlist = []
            for obj in ev:
                if obj is None:
                    newlist.append(None)
                elif obj == 'all_0':
                    newlist.extend(preprocessings[0:35])
                elif obj == 'sk_prep_all':      # no KernalCenter()
                    newlist.extend(preprocessings[0:7])
                elif obj == 'fs_all':
                    newlist.extend(preprocessings[7:14])
                elif obj == 'decomp_all':
                    newlist.extend(preprocessings[14:25])
                elif obj == 'k_appr_all':
                    newlist.extend(preprocessings[25:29])
                elif obj == 'reb_all':
                    newlist.extend(preprocessings[30:35])
                elif obj == 'imb_all':
                    newlist.extend(preprocessings[35:54])
                elif type(obj) is int and -1 < obj < len(preprocessings):
                    newlist.append(preprocessings[obj])
                elif hasattr(obj, 'get_params'):       # user uploaded object
                    if 'n_jobs' in obj.get_params():
                        newlist.append(obj.set_params(n_jobs=N_JOBS))
                    else:
                        newlist.append(obj)
                else:
                    sys.exit("Unsupported estimator type: %r" % (obj))

            search_params[param_name] = newlist

    return search_params

def main(inputs, infile_estimator, infile1, infile2,
         outfile_result, outfile_object=None,
         outfile_weights=None, groups=None,
         ref_seq=None, intervals=None, targets=None,
         fasta_path=None):
    """
    Parameter
    ---------
    inputs : str
        File path to galaxy tool parameter

    infile_estimator : str
        File path to estimator

    infile1 : str
        File path to dataset containing features

    infile2 : str
        File path to dataset containing target values

    outfile_result : str
        File path to save the results, either cv_results or test result

    outfile_object : str, optional
        File path to save searchCV object

    outfile_weights : str, optional
        File path to save model weights

    groups : str
        File path to dataset containing groups labels

    ref_seq : str
        File path to dataset containing genome sequence file

    intervals : str
        File path to dataset containing interval file

    targets : str
        File path to dataset compressed target bed file

    fasta_path : str
        File path to dataset containing fasta file
    """
    warnings.simplefilter('ignore')

    with open(inputs, 'r') as param_handler:
        params = json.load(param_handler)

    # conflict param checker
    if params['outer_split']['split_mode'] == 'nested_cv' \
            and params['save'] != 'nope':
        raise ValueError("Save best estimator is not possible for nested CV!")

    if not (params['search_schemes']['options']['refit']) \
            and params['save'] != 'nope':
        raise ValueError("Save best estimator is not possible when refit "
                         "is False!")

    params_builder = params['search_schemes']['search_params_builder']

    with open(infile_estimator, 'rb') as estimator_handler:
        estimator = load_model(estimator_handler)
    estimator_params = estimator.get_params()

    # store read dataframe object
    loaded_df = {}

    input_type = params['input_options']['selected_input']
    # tabular input
    if input_type == 'tabular':
        header = 'infer' if params['input_options']['header1'] else None
        column_option = (params['input_options']['column_selector_options_1']
                         ['selected_column_selector_option'])
        if column_option in ['by_index_number', 'all_but_by_index_number',
                             'by_header_name', 'all_but_by_header_name']:
            c = params['input_options']['column_selector_options_1']['col1']
        else:
            c = None

        df_key = infile1 + repr(header)
        df = pd.read_csv(infile1, sep='\t', header=header,
                         parse_dates=True)
        loaded_df[df_key] = df

        X = read_columns(df, c=c, c_option=column_option).astype(float)
    # sparse input
    elif input_type == 'sparse':
        X = mmread(open(infile1, 'r'))

    # fasta_file input
    elif input_type == 'seq_fasta':
        pyfaidx = get_module('pyfaidx')
        sequences = pyfaidx.Fasta(fasta_path)
        n_seqs = len(sequences.keys())
        X = np.arange(n_seqs)[:, np.newaxis]
        for param in estimator_params.keys():
            if param.endswith('fasta_path'):
                estimator.set_params(
                    **{param: fasta_path})
                break
        else:
            raise ValueError(
                "The selected estimator doesn't support "
                "fasta file input! Please consider using "
                "KerasGBatchClassifier with "
                "FastaDNABatchGenerator/FastaProteinBatchGenerator "
                "or having GenomeOneHotEncoder/ProteinOneHotEncoder "
                "in pipeline!")

    elif input_type == 'refseq_and_interval':
        path_params = {
            'data_batch_generator__ref_genome_path': ref_seq,
            'data_batch_generator__intervals_path': intervals,
            'data_batch_generator__target_path': targets
        }
        estimator.set_params(**path_params)
        n_intervals = sum(1 for line in open(intervals))
        X = np.arange(n_intervals)[:, np.newaxis]

    # Get target y
    header = 'infer' if params['input_options']['header2'] else None
    column_option = (params['input_options']['column_selector_options_2']
                     ['selected_column_selector_option2'])
    if column_option in ['by_index_number', 'all_but_by_index_number',
                         'by_header_name', 'all_but_by_header_name']:
        c = params['input_options']['column_selector_options_2']['col2']
    else:
        c = None

    df_key = infile2 + repr(header)
    if df_key in loaded_df:
        infile2 = loaded_df[df_key]
    else:
        infile2 = pd.read_csv(infile2, sep='\t',
                              header=header, parse_dates=True)
        loaded_df[df_key] = infile2

    y = read_columns(
            infile2,
            c=c,
            c_option=column_option,
            sep='\t',
            header=header,
            parse_dates=True)
    if len(y.shape) == 2 and y.shape[1] == 1:
        y = y.ravel()
    if input_type == 'refseq_and_interval':
        estimator.set_params(
            data_batch_generator__features=y.ravel().tolist())
        y = None
    # end y

    optimizer = params['search_schemes']['selected_search_scheme']
    optimizer = getattr(model_selection, optimizer)

    # handle gridsearchcv options
    options = params['search_schemes']['options']

    if groups:
        header = 'infer' if (options['cv_selector']['groups_selector']
                                    ['header_g']) else None
        column_option = (options['cv_selector']['groups_selector']
                                ['column_selector_options_g']
                                ['selected_column_selector_option_g'])
        if column_option in ['by_index_number', 'all_but_by_index_number',
                             'by_header_name', 'all_but_by_header_name']:
            c = (options['cv_selector']['groups_selector']
                        ['column_selector_options_g']['col_g'])
        else:
            c = None

        df_key = groups + repr(header)
        if df_key in loaded_df:
            groups = loaded_df[df_key]

        groups = read_columns(
                groups,
                c=c,
                c_option=column_option,
                sep='\t',
                header=header,
                parse_dates=True)
        groups = groups.ravel()
        options['cv_selector']['groups_selector'] = groups

    splitter, groups = get_cv(options.pop('cv_selector'))
    options['cv'] = splitter
    options['n_jobs'] = N_JOBS
    primary_scoring = options['scoring']['primary_scoring']
    options['scoring'] = get_scoring(options['scoring'])
    if options['error_score']:
        options['error_score'] = 'raise'
    else:
        options['error_score'] = np.NaN
    if options['refit'] and isinstance(options['scoring'], dict):
        options['refit'] = primary_scoring
    if 'pre_dispatch' in options and options['pre_dispatch'] == '':
        options['pre_dispatch'] = None

    # del loaded_df
    del loaded_df

    # handle memory
    memory = joblib.Memory(location=CACHE_DIR, verbose=0)
    # cache iraps_core fits could increase search speed significantly
    if estimator.__class__.__name__ == 'IRAPSClassifier':
        estimator.set_params(memory=memory)
    else:
        # For iraps buried in pipeline
        for p, v in estimator_params.items():
            if p.endswith('memory'):
                # for case of `__irapsclassifier__memory`
                if len(p) > 8 and p[:-8].endswith('irapsclassifier'):
                    # cache iraps_core fits could increase search
                    # speed significantly
                    new_params = {p: memory}
                    estimator.set_params(**new_params)
                # security reason, we don't want memory being
                # modified unexpectedly
                elif v:
                    new_params = {p, None}
                    estimator.set_params(**new_params)
            # For now, 1 CPU is suggested for iprasclassifier
            elif p.endswith('n_jobs'):
                new_params = {p: 1}
                estimator.set_params(**new_params)
            # for security reason, types of callbacks are limited
            elif p.endswith('callbacks'):
                for cb in v:
                    cb_type = cb['callback_selection']['callback_type']
                    if cb_type not in ALLOWED_CALLBACKS:
                        raise ValueError(
                            "Prohibited callback type: %s!" % cb_type)

    param_grid = _eval_search_params(params_builder)
    searcher = optimizer(estimator, param_grid, **options)

    # do nested split
    split_mode = params['outer_split'].pop('split_mode')
    # nested CV, outer cv using cross_validate
    if split_mode == 'nested_cv':
        outer_cv, _ = get_cv(params['outer_split']['cv_selector'])

        if options['error_score'] == 'raise':
            rval = cross_validate(
                searcher, X, y, scoring=options['scoring'],
                cv=outer_cv, n_jobs=N_JOBS, verbose=0,
                error_score=options['error_score'])
        else:
            warnings.simplefilter('always', FitFailedWarning)
            with warnings.catch_warnings(record=True) as w:
                try:
                    rval = cross_validate(
                        searcher, X, y,
                        scoring=options['scoring'],
                        cv=outer_cv, n_jobs=N_JOBS,
                        verbose=0,
                        error_score=options['error_score'])
                except ValueError:
                    pass
                for warning in w:
                    print(repr(warning.message))

        keys = list(rval.keys())
        for k in keys:
            if k.startswith('test'):
                rval['mean_' + k] = np.mean(rval[k])
                rval['std_' + k] = np.std(rval[k])
            if k.endswith('time'):
                rval.pop(k)
        rval = pd.DataFrame(rval)
        rval = rval[sorted(rval.columns)]
        rval.to_csv(path_or_buf=outfile_result, sep='\t',
                    header=True, index=False)
    else:
        if split_mode == 'train_test_split':
            train_test_split = try_get_attr(
                'galaxy_ml.model_validations', 'train_test_split')
            # make sure refit is choosen
            # this could be True for sklearn models, but not the case for
            # deep learning models
            if not options['refit'] and \
                    not all(hasattr(estimator, attr)
                            for attr in ('config', 'model_type')):
                warnings.warn("Refit is change to `True` for nested "
                              "validation!")
                setattr(searcher, 'refit', True)
            split_options = params['outer_split']

            # splits
            if split_options['shuffle'] == 'stratified':
                split_options['labels'] = y
                X, X_test, y, y_test = train_test_split(X, y, **split_options)
            elif split_options['shuffle'] == 'group':
                if groups is None:
                    raise ValueError("No group based CV option was "
                                     "choosen for group shuffle!")
                split_options['labels'] = groups
                if y is None:
                    X, X_test, groups, _ =\
                        train_test_split(X, groups, **split_options)
                else:
                    X, X_test, y, y_test, groups, _ =\
                        train_test_split(X, y, groups, **split_options)
            else:
                if split_options['shuffle'] == 'None':
                    split_options['shuffle'] = None
                X, X_test, y, y_test =\
                    train_test_split(X, y, **split_options)
        # end train_test_split

        # shared by both train_test_split and non-split
        if options['error_score'] == 'raise':
            searcher.fit(X, y, groups=groups)
        else:
            warnings.simplefilter('always', FitFailedWarning)
            with warnings.catch_warnings(record=True) as w:
                try:
                    searcher.fit(X, y, groups=groups)
                except ValueError:
                    pass
                for warning in w:
                    print(repr(warning.message))

        # no outer split
        if split_mode == 'no':
            # save results
            cv_results = pd.DataFrame(searcher.cv_results_)
            cv_results = cv_results[sorted(cv_results.columns)]
            cv_results.to_csv(path_or_buf=outfile_result, sep='\t',
                              header=True, index=False)

        # train_test_split, output test result using best_estimator_
        # or rebuild the trained estimator using weights if applicable.
        else:
            scorer_ = searcher.scorer_
            if isinstance(scorer_, collections.Mapping):
                is_multimetric = True
            else:
                is_multimetric = False

            best_estimator_ = getattr(searcher, 'best_estimator_', None)
            if not best_estimator_:
                raise ValueError("GridSearchCV object has no "
                                 "`best_estimator_` when `refit`=False!")

            if best_estimator_.__class__.__name__ == 'KerasGBatchClassifier' \
                    and hasattr(estimator.data_batch_generator, 'target_path'):
                test_score = best_estimator_.evaluate(
                    X_test, scorer=scorer_, is_multimetric=is_multimetric)
            else:
                test_score = _score(best_estimator_, X_test,
                                    y_test, scorer_,
                                    is_multimetric=is_multimetric)

            if not is_multimetric:
                test_score = {primary_scoring: test_score}
            for key, value in test_score.items():
                test_score[key] = [value]
            result_df = pd.DataFrame(test_score)
            result_df.to_csv(path_or_buf=outfile_result, sep='\t',
                             header=True, index=False)

    memory.clear(warn=False)

    if outfile_object:
        best_estimator_ = getattr(searcher, 'best_estimator_', None)
        if not best_estimator_:
            warnings.warn("GridSearchCV object has no attribute "
                          "'best_estimator_', because either it's "
                          "nested gridsearch or `refit` is False!")
            return

        main_est = best_estimator_
        if isinstance(best_estimator_, pipeline.Pipeline):
            main_est = best_estimator_.steps[-1][-1]

        if hasattr(main_est, 'model_') \
                and hasattr(main_est, 'save_weights'):
            if outfile_weights:
                main_est.save_weights(outfile_weights)
            del main_est.model_
            del main_est.fit_params
            del main_est.model_class_
            del main_est.validation_data
            if getattr(main_est, 'data_generator_', None):
                del main_est.data_generator_

        with open(outfile_object, 'wb') as output_handler:
            pickle.dump(best_estimator_, output_handler,
                        pickle.HIGHEST_PROTOCOL)

if __name__ == '__main__':
    aparser = argparse.ArgumentParser()
    aparser.add_argument("-i", "--inputs", dest="inputs", required=True)
    aparser.add_argument("-e", "--estimator", dest="infile_estimator")
    aparser.add_argument("-X", "--infile1", dest="infile1")
    aparser.add_argument("-y", "--infile2", dest="infile2")
    aparser.add_argument("-O", "--outfile_result", dest="outfile_result")
    aparser.add_argument("-o", "--outfile_object", dest="outfile_object")
    aparser.add_argument("-w", "--outfile_weights", dest="outfile_weights")
    aparser.add_argument("-g", "--groups", dest="groups")
    aparser.add_argument("-r", "--ref_seq", dest="ref_seq")
    aparser.add_argument("-b", "--intervals", dest="intervals")
    aparser.add_argument("-t", "--targets", dest="targets")
    aparser.add_argument("-f", "--fasta_path", dest="fasta_path")
    args = aparser.parse_args()

    main(args.inputs, args.infile_estimator, args.infile1, args.infile2,
         args.outfile_result, outfile_object=args.outfile_object,
         outfile_weights=args.outfile_weights, groups=args.groups,
         ref_seq=args.ref_seq, intervals=args.intervals,
         targets=args.targets, fasta_path=args.fasta_path)


import random

import matplotlib.pyplot as plt

## Part d
from scipy import stats
import numpy as np
import math
import matplotlib.pyplot as plt
import seaborn as sns

# 1.c Simple Random Walk
def get_next_position(current_position, step, length, boundary_conditions):

	if boundary_conditions == "unlimited":
		return current_position + step

	elif boundary_conditions == "periodic":
		# Subtract and then add 1 to deal with python indexing at 0 while still making use of modulus function
		return (current_position + step-1)%(length) + 1

	elif boundary_conditions == "reflecting":
		if current_position == length:
			return length -1
		elif current_position == 1:
			return 2
		else:
			return current_position + step

	elif boundary_conditions == "closed":
		next_position = current_position + step
		if next_position > length:
			return length
		elif next_position < 1:
			return 1
		else:
			return next_position

	elif boundary_conditions == "absorbing":
		if current_position == length:
			return length
		elif current_position == 1:
			return 1
		else:
			return current_position + step

def simple_random_walk(p, total_steps, length, boundary_conditions):

	positions = [1]
	steps = []

	for i in range(total_steps):
		if random.random() < p:
			step = 1
		else:
			step = -1

		steps.append(step)
		current_position = positions[i]
		next_position = get_next_position(current_position, step, length, boundary_conditions)
		positions.append(next_position)

	return positions

def plot_distribution_500_realisations(total_steps):

	p = 0.65
	realisations = 500
	observations = np.zeros(realisations)
	for i in range(realisations):
		final_position = simple_random_walk(p, total_steps, 10, "closed")[-1]
		observations[i] = final_position

	scaled_observations = (observations-1) * 10/9

	states = [1,2,3,4,5,6,7,8,9,10]
	stat_dist = [(p/(1-p))**x for x in states]
	total_stat_dist = sum(stat_dist)
	stat_dist_norm = [x/total_stat_dist for x in stat_dist]
	plt.hist(scaled_observations, density=True, bins=10, label="Empirical density distribution")
	plt.plot(states, stat_dist_norm, 'ro', label="Theoretical stationary distribution")

	plt.title('Closed SRW - {} timesteps, L=10, p=0.65'.format(total_steps,p), fontsize = 16)

	plt.legend()
	plt.xlabel('x', fontsize = 20)
	plt.ylabel('frequency', fontsize = 20)

	plt.savefig('{}_steps.png'.format(total_steps))

	plt.show()

def plot_distribution_1_realisation_500_steps(title):

	p = 0.65

	positions = simple_random_walk(p, 500, 10, "closed")
	np_positions = np.array(positions)
	scaled_positions = (np_positions-1) * 10/9

	plt.hist(scaled_positions, density=True, label="Empirical density distribution")

	states = [1,2,3,4,5,6,7,8,9,10]
	stat_dist = [(p/(1-p))**x for x in states]
	total_stat_dist = sum(stat_dist)
	stat_dist_norm = [x/total_stat_dist for x in stat_dist]
	plt.plot(states, stat_dist_norm, 'ro',label="Theoretical stationary distribution")

	plt.legend(loc=2, prop={'size': 6})

	plt.title('Closed SRW. 1 realisation. 500 timesteps, L=10, p=0.65')

	plt.xlabel('x', fontsize = 10)
	plt.ylabel('frequency', fontsize = 12)
	plt.savefig('500_steps.png')

	plt.show()

plot_distribution_1_realisation_500_steps(1)

def plot_4_realisations():

	plt.rcParams['figure.figsize'] = [10, 6]

	plt.figure(1)
	plt.suptitle('Closed SRW. 4 seperate realisations. 500 timesteps, L=10, p=0.6', fontsize = 12)

	plt.subplot(221)
	plt.axis((0,10,0,0.5))
	plot_distribution_1_realisation_500_steps("a")
	plt.subplot(222)

	plt.axis((0,10,0,0.5))

	plot_distribution_1_realisation_500_steps("b")
	plt.subplot(223)

	plt.axis((0,10,0,0.5))

	plot_distribution_1_realisation_500_steps("c")
	plt.subplot(224)

	plt.axis((0,10,0,0.5))

	plot_distribution_1_realisation_500_steps("d")

	plt.savefig("500_steps_4.png")
	plt.show()

def one_realisation(mu, sigma, tmax):

	#np.random.seed(12)

	#Generate sequences of random variables
	x_sequence = np.random.normal(mu, sigma, tmax)
	#x_sequence = np.random.randn(tmax)*sigma + mu # How
	y_sequence = np.cumsum(x_sequence)
	z_sequence = np.exp(y_sequence)

	return y_sequence, z_sequence

def plot_y_and_z(mu, sigma, tmax):

	y_sequence, z_sequence = one_realisation(mu, sigma, tmax)

	plt.plot(range(tmax), z_sequence, label = r'$Z_n$')
	plt.plot(range(tmax), y_sequence, label = r'$Y_n$')
	plt.legend(loc = 'upper left', fontsize = 20)
	plt.xlabel('n', fontsize=20)
	plt.show()

def empirical_results(mu, sigma, tmax, realisations=500):

	results = np.zeros((realisations,tmax))

	for realisation in range(realisations):
		results[realisation,:] = one_realisation(mu,sigma,tmax)[1]

	return results

def plot_empirical_results():

	mu = 0
	sigma = 0.2
	tmax = 100

	plt.rcParams['figure.figsize'] = [10, 6]

	results = empirical_results(mu,sigma,tmax)

	empirical_averages = results.mean(axis=0)
	empirical_sds = results.std(axis=0)/np.sqrt(500)

	plt.figure(0)
	plt.errorbar(range(tmax), empirical_averages, yerr=empirical_sds, label = r'Emprical Average and standard deviation')

	plt.legend(loc="upper left")
	plt.xlabel('n', fontsize=20)
	plt.ylabel(r'$Z_n$', fontsize=20)

	plt.title(r'Emprical Average of $Z_n$. 500 realisations, $\mu_x=0$, $\sigma_x=0.2$')

	plt.savefig('Emprical Average.png')

	plt.show()

def box_plots():

	mu = 0
	sigma = 0.2
	tmax = 100

	plt.rcParams['figure.figsize'] = [10, 6]

	results = empirical_results(mu,sigma,tmax)

	timestep_10_results = results[:,[9]]

	f, (ax1, ax2) = plt.subplots(1, 2)

	ax1.boxplot(timestep_10_results)
	ax1.set_yscale("log")
	ax1.set_title("n=10")

	ax1.set_ylabel(r'$log(Z_n)$')

	ax2.boxplot(results[:,[99]])
	#ax2.boxplot(results[:,[99]], whis=100) # Plot whiskers right up to all outlying points
	ax2.set_title("n=100")

	ax2.set_ylabel(r'$log(Z_n)$')
	ax2.set_yscale("log")

	plt.suptitle(r"Box plots of $Z_n$ at n=10 and n=100. $\mu_x=0$, $\sigma_x=0.2$")
	plt.savefig('box_plots.png')

	plt.show()

def empirical_pdf_at_timestep_original(timestep):

	mu = 0
	sigma = 0.2
	tmax = 100

	results = empirical_results(mu,sigma,tmax)
	timestep_results = results[:,[timestep-1]]
	#plt.hist(timestep_results, density=True)

	# KDE plot 1
	kde = stats.gaussian_kde(timestep_results.reshape(500))
	xx = np.linspace(0,10,1000)
	#plt.plot(xx, kde(xx))

	# Theoretical plot 1
	rv = stats.lognorm([(sigma)*(timestep**0.5)], scale=np.exp(timestep*mu))
	plt.plot(xx, rv.pdf(xx))

	# KDE plot 2
	sns.kdeplot(timestep_results.reshape(500), gridsize=10000 )

	x1,x2,y1,y2 = plt.axis()
	plt.axis((0,10,y1,y2))

	plt.show()

def empirical_pdf_at_timestep(timestep):

	mu = 0
	sigma = 0.2
	tmax = 100

	plt.rcParams['figure.figsize'] = [10, 6]

	results = empirical_results(mu,sigma,tmax, 5000)
	timestep_results = results[:,[timestep-1]]
	#plt.hist(timestep_results, density=True)

	# KDE plot 1
	kde = stats.gaussian_kde(timestep_results.reshape(5000))
	xx = np.linspace(0,10,1000)
	#plt.plot(xx, kde(xx))

	# Theoretical plot 1
	rv = stats.lognorm([(sigma)*(timestep**0.5)], scale=np.exp(timestep*mu))
	plt.plot(xx, rv.pdf(xx), label="Theoretical Distribution")

	# KDE plot 2
	sns.kdeplot(timestep_results.reshape(5000), gridsize=10000, label="Empirical KDE")

	if timestep > 10:
		x1,x2,y1,y2 = plt.axis()
		plt.axis((-1,10,0,0.1))

	plt.xlabel(r'$Z_n$')
	plt.ylabel("Probablity Density")
	plt.title(r"Emprical KDE and Theoretical PDF of $Z_n$ at n={}. $\mu_x=0, \sigma_x=0.2$".format(timestep))
	plt.legend()

	#plt.savefig('emprical_pdf_kde_{}.png'.format(timestep))

	plt.show()

def empirical_pdf_at_timestep_b(timestep):

	mu = 0
	sigma = 0.2
	tmax = 100

	results = empirical_results(mu,sigma,tmax)
	timestep_results = results[:,[timestep-1]]
	#plt.hist(timestep_results, density=True)

	# KDE plot 1
	kde = stats.gaussian_kde(timestep_results.reshape(500))
	xx = np.linspace(0,10,1000)
	#plt.plot(xx, kde(xx))

	# KDE plot 2
	sns.kdeplot(timestep_results.reshape(500), gridsize=10000 )

	# Theoretical plot 1
	rv = stats.lognorm([(sigma)*(timestep**0.5)], scale=np.exp(timestep*mu))
	plt.plot(xx, rv.pdf(xx))

	x1,x2,y1,y2 = plt.axis()
	plt.axis((0,10,0,0.4))

	plt.show()

def log_normal_pdf(z_input, mu, sigma, timestep):

	pdf_sequence = []

	for z in z_input:

		if z == 0:
			pdf_sequence.append(0)
		else:
			exp_numerator = -1*((math.log(z) - timestep*mu)**2)
			exp_denominator = 2 * timestep * sigma**2
			answer = (1/z) * 1/(sigma) * 1/((2*math.pi*timestep)**0.5)* np.exp(exp_numerator/exp_denominator)
			pdf_sequence.append(answer)
	return pdf_sequence

def ergodic_average():

	mu = 0
	sigma = 0.2
	tmax = 100

	result = one_realisation(mu,sigma,tmax)[1]

	ergodic_totals = np.cumsum(result)
	n_sequence = np.array(range(tmax))+1
	ergodic_averages = np.divide(ergodic_totals, n_sequence)

	plt.plot(range(tmax), ergodic_averages)

	plt.ylabel(r'$\bar{\mu}_N$')
	plt.xlabel("N")
	plt.title(r"Ergodic Average of $Z_n$ for 1 realisation. $\mu_x=0, \sigma_x=0.2$")

	plt.savefig('ergodic_average.png')

	plt.show()

def empirical_tail(timestep):

	mu = -0.02
	sigma = 0.2
	tmax = 100

	results = empirical_results(mu,sigma,tmax)

	timestep_results = results[:,[timestep-1]].reshape(500,)

	data_size=len(timestep_results)

	# Set bins edges
	data_set=sorted(set(timestep_results))
	bins=np.append(data_set, data_set[-1]+1)

	# Use the histogram function to bin the data
	counts, bin_edges = np.histogram(timestep_results, bins=bins, density=False)

	counts=counts.astype(float)/data_size

	# Find the cdf
	cdf = np.cumsum(counts)

	# Plot the cdf
	plt.plot(bin_edges[0:-1], np.ones(len(cdf))-cdf,linestyle='--', marker="o", color='b', markersize=1, label="Empirical Tail")
	plt.ylim((0,1))
	plt.ylabel("1-CDF")
	plt.xlabel(r"$Z_n$")
	plt.yscale("linear")
	plt.xscale("log")  # Q _ what axis scales are most informative?
	plt.grid(True)

	rv = stats.lognorm([(sigma)*(timestep**0.5)], scale=np.exp(timestep*mu))
	plt.plot(bin_edges[0:-1], rv.sf(bin_edges[0:-1]), color="red", label="Theoretical Survival Function")
	plt.legend()

	plt.title(r"Empirical Tail of $Z_n$ at n={}. 500 realisations, $\mu_x=-0.02, \sigma_x=0.2$".format(timestep))
	plt.savefig('empirical_tail_{}.png'.format(timestep))

	plt.show()


import sys

import os
import zipfile
import tempfile
import sched, time
import numpy as np

from b3w import B3W
from glob import glob
from osgeo import gdal
from shutil import rmtree
from subprocess import Popen, PIPE, STDOUT
from typing import Any, List, Tuple, Set

from sentinel import Config
from sentinel import DataHub
from sentinel import Polygons
from utils import get_environment, print_snapshots, remove

# Main loop period (seconds)
INTERVAL: int = 10#0

def check_in_aws(s3: B3W, prefix: str, depth: int = 1) -> Set[str]:
    objects: Set[str] = set() #List[str] = []
    try:
        for s3o in s3.ls(prefix):
            if prefix:
                s3o = s3o.replace(prefix.strip('/'), '').strip('/')
            if not s3o:
                continue
            objects.add('/'.join(s3o.split('/')[:depth]))
    except Exception as e:
        #TODO: debug and handle different kinds of exceptions
        raise e

    return objects

def get_from_aws(s3: B3W, prefix: str,
        path: str = '/dev/shm/gps/input'
    ) -> List[str]:
    files: List[str] = []
    try:
        #objects = s3.ls(prefix)
        #print(f"DEBUG: objects = {objects}")
        extensions = ('.geojson', '.shp', '.shx', '.dbf', '.prj', '.cpg')
        for s3o in s3.ls(prefix):
            #print(f"DEBUG: get_from_aws s3o = {s3o}")
            if prefix:
                filename = s3o.replace(prefix.strip('/'), '').strip('/')
            else:
                filename = s3o
            if not filename.endswith(extensions):
                continue
            #filename = os.path.join(path, *filename.lstrip('/').split('/'))
            filename = os.path.join(path, *filename.split('/'))
            #print(f"DEBUG: {s3o} -> {filename}")
            os.makedirs(os.path.dirname(filename), exist_ok=True)
            s3.get(s3o, filename)
            files.append(filename)
    except Exception as e:
        #TODO: debug and handle different kinds of exceptions
        raise e

    return files

def put_to_aws(s3: B3W, prefix: str,
        path: str = '/dev/shm/gps/output'
    ) -> List[str]:
    objects: List[str] = []
    try:
        path = os.path.normpath(path)
        for filename in glob(os.path.join(path, '**'),  recursive=True):
            if not os.path.isfile(filename):
                continue
            s3o = os.path.relpath(filename, path)
            if prefix:
                s3o = '/'.join([prefix.strip('/'), *s3o.split(os.path.sep)])
            else:
                s3o = '/'.join([*s3o.split(os.path.sep)])
            #print(f"DEBUG: '{filename}' -> '{s3o}'")
            s3.put(filename, s3o, force=True)
    except Exception as e:
        #TODO: debug and handle different kinds of exceptions
        raise e

    return objects

def remove_from_aws(s3: B3W, prefix: str) -> List[str]:
    # TODO: remove this function (DEBUG only)
    objects: List[str] = []
    try:
        #objects = s3.ls(prefix)
        #print(f"DEBUG: objects to remove = {objects}")
        for s3o in s3.ls(prefix):
            s3._B3W__s3r.Object(s3._B3W__bucket_name, s3o).delete()
            print(f"DEBUG: removed S3 object '{s3o}'")
            objects.append(s3o)
    except Exception as e:
        print(f"Error removing S3 object:\n{e}")

    return objects

def sync_with_aws(
        s3: B3W, prefix: str, data_hub: DataHub,
        snapshot: Any, path: str = '/dev/shm/gps/data'
    ) -> str:
    filename: str = None
    try:
        data_hub.config.output = os.path.join(path, snapshot.uuid)
        os.makedirs(data_hub.config.output, exist_ok=True)
        objects = s3.ls('/'.join([prefix, snapshot.uuid]))
        print(f"DEBUG: S3 objects = {objects}")
        #if len(objects) > 0:
        for s3o in objects:
            #s3o = s3o.replace(prefix, '').lstrip('/')
            if s3o.strip('/').endswith(snapshot.uuid):
                continue
            filename = os.path.join(path, s3o.replace(prefix, '').lstrip('/'))
            print(f"DEBUG: syncing '{s3o}' -> '{filename}'")
            s3.get(s3o, filename)
            break
        if not filename:
            data_hub.download(snapshot)
            #print(f"DEBUG: data_hub.config.output = {data_hub.config.output}")
            for filename in glob(os.path.join(data_hub.config.output, '*')):
            #print(f"DEBUG: filename = {filename}")
            #if len(filename) < 1:
            #    raise FileNotFoundError(f"failed to download {snapshot.uuid}!")
            #else:
            #    filename = filename[0]
                s3o = filename.replace(path, '').lstrip(os.path.sep)
                # TODO: check filename against snapshot.uuid
                s3o = '/'.join([prefix, s3o])
                print(f"DEBUG: syncing '{filename}' -> '{s3o}'")
                s3.put(filename, s3o)
                break
            if not filename:
                print(f"failed to download {snapshot.uuid}!")
        data_hub.config.output = None # TODO: devise something smart
    except FileNotFoundError as e:
        print(f"Failure: {e}")
    except Exception as e:
        #TODO: debug and handle different kinds of exceptions
        raise e

    return filename

def set_debug_aws() -> None:
    s3_id, s3_key, s3_bucket, s3_input, s3_output, s3_sync = get_environment()
    path_input, path_output = ('/dev/shm/gps/input', '/dev/shm/gps/output')
    path_data = '/dev/shm/gps/data'
    #print(f"DEBUG:\n\t{'S3_ID'.ljust(16)}{s3_id}"
    #      f"\n\t{'S3_KEY'.ljust(16)}{s3_key}"
    #      f"\n\t{'S3_BUCKET'.ljust(16)}{s3_bucket}"
    #      f"\n\t{'S3_INPUT'.ljust(16)}{s3_input}"
    #      f"\n\t{'S3_OUTPUT'.ljust(16)}{s3_output}"
    #      f"\n\t{'S3_SYNC'.ljust(16)}{s3_sync}")
    s3 = B3W(s3_bucket, s3_id, s3_key)
    # DEBUG: put test data to S3
    remove_from_aws(s3, s3_input + '/test')
    remove_from_aws(s3, s3_output + '/test')
    for filename in glob(os.path.join('data', '*')):
        #s3.put('data/geotiff.xml', '/'.join([s3_input, 'test', 'geotiff.xml']))
        s3o = '/'.join([s3_input, 'test', os.path.basename(filename)])
        s3.put(filename, s3o)
        print(f"DEBUG: '{filename}' -> '{s3o}'")
    #print(s3.ls(s3_input))
    #sys.exit(0)

    return None

def process_sentinel1(filename, path_output, area, shapes):
    title = os.path.splitext(os.path.basename(filename))[0]
    options_warp = {
            'format': 'GTiff',
            'dstSRS': 'EPSG:32640',
            'creationOptions': ['COMPRESS=DEFLATE'],
            'xRes': 40,
            'yRes': 40
    }
    with tempfile.TemporaryDirectory() as path_temp:
        with zipfile.ZipFile(filename, 'r') as archive:
            archive.extractall(path_temp)
            path_safe = glob(os.path.join(path_temp, f"*.SAFE"))[0]
            dataset = gdal.Open(path_safe, gdal.GA_ReadOnly)
            subsets = dataset.GetSubDatasets()
            datasets = {}
            for i, p in enumerate(['HH', 'HV', 'RGB']):
                print(f"Reading {subsets[i][1]}...")
                datasets[p] = gdal.Open(subsets[i][0], gdal.GA_ReadOnly)
            filenames = []
            name_area = os.path.splitext(os.path.basename(area))[0]
            print(f"Warping polarizations...")
            for name, source in datasets.items():
                #
                # Prepare filenames and paths
                #
                for shape in shapes:
                    if shape:
                        name_shape = os.path.basename(shape)
                        name_shape = os.path.splitext(name_shape)[0]
                        data_prefix = f"{name_area}_{name_shape}"
                        options_cutline = {'cutlineDSName': shape,
                                           'cropToCutline': True}
                    else:
                        data_prefix = f"{name_area}"
                        options_cutline = {}
                    data_output = os.path.join(path_output, data_prefix)
                    if not name in ['RGB', 'INV']:
                        data_output = os.path.join(data_output, name.lower())
                        os.makedirs(data_output, exist_ok=True)
                        print(f"{data_output.replace(path_output, '')}")
                        destination = f"{os.path.join(data_output, title)}.tiff"
                        filenames.append(destination)
                        gdal.Warp(destination, source, **options_warp,
                                  **options_cutline)
                    else:
                        # Basic RGB processing
                        driver_mem = gdal.GetDriverByName('MEM')
                        memoset = driver_mem.CreateCopy('', source, 0)
                        band_hh = memoset.GetRasterBand(1)
                        image_hh = band_hh.ReadAsArray()
                        mask_hh = image_hh == 0
                        image_hh = np.ma.array(image_hh, mask=mask_hh,
                                               dtype=np.float32)
                        del mask_hh
                        band_hv = memoset.GetRasterBand(2)
                        image_hv = band_hv.ReadAsArray()
                        mask_hv = image_hv == 0
                        image_hv = np.ma.array(image_hv, mask=mask_hv,
                                               dtype=np.float32)
                        del mask_hv
                        stats_hh = (image_hh.mean().astype(np.float32),
                                    image_hh.std().astype(np.float32))
                        stats_hv = (image_hv.mean().astype(np.float32),
                                    image_hv.std().astype(np.float32))
                        image_hh = np.ma.tanh(image_hh / (stats_hh[0] + 2 *
                                                          stats_hh[1]))
                        image_hv = np.ma.tanh(image_hv / (stats_hv[0] + 2 *
                                                          stats_hv[1]))
                        image_ratio = image_hh / image_hv
                        stats_ratio = (image_ratio.mean().astype(np.float32),
                                       image_ratio.std().astype(np.float32))
                        image_ratio = image_ratio / image_ratio.max()
                        image_negative = (np.float32(1) -
                                          np.ma.tanh(image_hh / image_hv))
                        # Convert to byte type
                        image_hh = (image_hh * 254 + 1).astype(np.uint8)
                        image_hv = (image_hv * 254 + 1).astype(np.uint8)
                        image_ratio = (image_ratio * 254 + 1).astype(np.uint8)
                        image_negative = (image_negative * 254 + 1)\
                                         .astype(np.uint8)
                        # Write channels to the MEM dataset
                        memoset.AddBand()
                        band_ex = memoset.GetRasterBand(3)
                        band_ex.SetColorInterpretation(gdal.GCI_BlueBand)
                        band_hh.WriteArray(image_hh)
                        band_hh.SetColorInterpretation(gdal.GCI_RedBand)
                        band_hv.WriteArray(image_hv)
                        band_hv.SetColorInterpretation(gdal.GCI_GreenBand)
                        # Create ratio band (HH, HV, HH/HV)
                        band_ex.WriteArray(image_ratio)
                        band_ex.SetMetadata({'POLARISATION': 'HH/HV',
                                             'SWATH': 'EW'})
                        path_ratio = os.path.join(data_output, 'ratio')
                        os.makedirs(path_ratio, exist_ok=True)
                        print(f"{path_ratio.replace(path_output, '')}")
                        destination = f"{os.path.join(path_ratio, title)}.tiff"
                        filenames.append(destination)
                        gdal.Warp(destination, memoset, **options_warp,
                                  outputType=gdal.GDT_Byte, **options_cutline)
                        # Create negative band (HH, HV, 1 - HH/HV)
                        band_ex.WriteArray(image_negative)
                        band_ex.SetMetadata({'POLARISATION': '1 - HH/HV',
                                             'SWATH': 'EW'})
                        path_negative = os.path.join(data_output, 'negative')
                        os.makedirs(path_negative, exist_ok=True)
                        print(f"{path_negative.replace(path_output, '')}")
                        destination = f"{os.path.join(path_negative, title)}.tiff"
                        filenames.append(destination)
                        gdal.Warp(destination, memoset, **options_warp,
                                  outputType=gdal.GDT_Byte, **options_cutline)
    print(f"Done!")
    return filenames

def process_sentinel2(filename, path_output, area, shapes):
    title = os.path.splitext(os.path.basename(filename))[0]
    dataset = gdal.Open(filename, gdal.GA_ReadOnly)
    subsets = dataset.GetSubDatasets()
    assert len(subsets) > 0, f"no sub datasets found!"
    dataset = gdal.Open(subsets[0][0], gdal.GA_ReadOnly)
    #print(f"{snapshot.title} -->")
    print(f"Reading {subsets[0][1][:1].lower()}",
          f"{subsets[0][1][1:]}", sep='')
    image = dataset.ReadAsArray()
    if image.ndim < 3:
        image = image[None, ...]
    image = np.moveaxis(image[:3, ...], 0, -1) # CHW -> HWC
    image = image.astype(np.float32)
    print(f"Calculating optimal histogram...")
    clip = image.mean().astype(np.float32) * np.float32(2)
    image = image / clip
    image = np.tanh(image)
    # Apply gamma correction here (TODO)
    image = (image * 254 + 1).round().astype(np.uint8)
    # Apply nodata mask here (TODO)
    image = np.moveaxis(image, -1, 0) # HWC -> CHW
    print(f"Applying histogram...")
    tempset = gdal.GetDriverByName('MEM')\
              .CreateCopy('', dataset, 0)
    for i in range(image.shape[0]):
        band = tempset.GetRasterBand(i + 1)
        band.WriteArray(image[i].astype(np.uint16))
        del band
    print(f"Writing to temporary file...")
    with tempfile.TemporaryDirectory() as path_temp:
        temp = os.path.join(path_temp, 'temp.tiff')
        gdal.Translate(temp, tempset,
                       creationOptions=['COMPRESS=DEFLATE'],
                       format='GTiff', bandList=[1, 2, 3],
                       outputType=gdal.GDT_Byte)
        del tempset
        #
        # Prepare filenames and paths
        #
        filenames = []
        name_area = os.path.splitext(os.path.basename(area))[0]
        for shape in shapes:
            if shape:
                name_shape = os.path.basename(shape)
                name_shape = os.path.splitext(name_shape)[0]
                data_prefix = f"{name_area}_{name_shape}"
                options = {'cutlineDSName': shape,
                           'cropToCutline': True}
            else:
                data_prefix = f"{name_area}"
                options = {}
            data_output = os.path.join(path_output,
                                       #data_name,
                                       data_prefix)
            os.makedirs(data_output, exist_ok=True)
            destination = f"{os.path.join(data_output, title)}.tiff"
            filenames.append(destination)
            gdal.Warp(destination, temp, **options)
    print(f"Done!")
    return filenames

def main(periodic: sched.scheduler) -> None:
    # Set working variables
    s3_id, s3_key, s3_bucket, s3_input, s3_output, s3_sync = get_environment()
    path_input, path_output = ('/dev/shm/gps/input', '/dev/shm/gps/output')
    path_data = '/dev/shm/gps/data'

    #print(f"\n=== Started input processing cycle ===\n")
    s3 = B3W(s3_bucket, s3_id, s3_key)

    # Get input files from S3
    files_input = get_from_aws(s3, s3_input, path_input)
    #print("DEBUG: input files -->")
    #print("\n".join([f"DEBUG: {filename}" for filename in files_input]))
    objects_output = check_in_aws(s3, s3_output, depth=1)
    #print("DEBUG: output sets -->")
    #print("\n".join([f"DEBUG: {name}" for name in objects_output]))
    # DEBUG: list sync objects in S3, remove output test set
    #objects_sync = check_in_aws(s3, s3_sync) # don't uncomment - dangerous!
    #print("DEBUG: sync objects -->")
    #print("\n".join([f"DEBUG: {name}" for name in objects_sync]))

    # Initialize Copernicus Open Data Access Hub search object
    config = Config.load('config.yaml')
    data_hub = DataHub(config, limit=1000)

    # Cycle through all the data input sets: a set may contain multiple
    # input areas and shapes to process. Result will be a snapshot that is
    # cut with each shape (if any)
    for data_input in glob(os.path.join(path_input, '*')):
        if not os.path.isdir(data_input):
            #print(f"DEBUG: '{data_input}' is not a valid data input!")
            #print("TODO: unzip archived input sets...")
            continue
        data_name = os.path.basename(data_input)
        #print(f"DEBUG: 'data_input' basename = {data_name}")
        if data_name in objects_output:
            #print(f"Output set for '{data_input}' already exists. Skipping...")
            continue
        #print(f"DEBUG: input directory --->\n{os.listdir(data_input)}\n")
        areas = glob(os.path.join(data_input, '*.geojson'))
        shapes = glob(os.path.join(data_input, '*.shp'))
        #print(f"DEBUG: shapes = {shapes}")
        if not shapes:
            shapes.append(None)
        for area in areas:
            try:
                print(f"\n=== Processing '{area}' ===\n")
                polygon, properties = Polygons.read_geojson(area)
            except Exception as e:
                print(f"Failed to read '{area}'!\n{str(e)}")
                continue
            #print(f"DEBUG:\n{polygon}")

            # Set config key (search area)
            #print(f"DEBUG: config.search -->\n{config.search}")
            search = config.search.copy()
            search.update(properties)
            #config.search["footprint"] = f"\"Intersects({polygon})\""
            #print(f"DEBUG: config.search -->\n{config.search}")
            #print(f"Config 'search' section:\n{config.search}")

            snapshots = data_hub.search(search, area=polygon)
            snapshots = sorted(snapshots,
                               key=lambda item: item.begin_position)

            print(f"\n=== {len(snapshots)} snapshots found ===\n")
            # print_snapshots(snapshots) # DEBUG
            # break # DEBUG

            print(f"\n=== Processing snapshots and shapes ===\n")
            for index, snapshot in enumerate(snapshots):
                filename = sync_with_aws(s3, s3_sync, data_hub, snapshot,
                                         path_data)
                if not filename:
                    print(f"'\n{snapshot.uuid}' not synced. Skipping...")
                    continue
                else:
                    print(f"\n{index:8d}: {snapshot.title}")
                try:
                    # Process each superposition of an area and a shape
                    #
                    # Process a snapshot
                    #
                    #print(f"DEBUG: search keys = {search.keys()}")
                    path_target = os.path.join(path_output, data_name)
                    #print(f"DEBUG: path_data = '{path_data}'")
                    if search['platformName'] == 'Sentinel-2':
                        filenames = process_sentinel2(filename, path_target,
                                                      area, shapes)
                    elif search['platformName'] == 'Sentinel-1':
                        filenames = process_sentinel1(filename, path_target,
                                                      area, shapes)
                    else:
                        filenames = []
                        print(f"NOT IMPLEMENTED: {snapshot.title}",
                              f"{config.search['platformName']}")
                    #print(f"DEBUG: exporting '{data_prefix}' to S3 -->")
                    # Put processing result (for each output set) to S3
                    result = put_to_aws(s3, s3_output, path_output) # result...
                    for outfile in filenames:
                        remove(outfile) # all files (TODO: file or directory)
                except Exception as e:
                    print(f"FAILED: {e}")
                    raise e
                remove(filename) # remove snapshot
                #break # DEBUG: the first snapshot only
            print(f"\n=== Done snapshots for '{area}' ===\n")
        # Clean up output set (there should remain only logs)
        try:
            rmtree(os.path.join(path_output, data_name)) # data output - prefix
        except FileNotFoundError as e:
            pass
    # Clean up
    for path in (path_data, path_input, path_output):
        try:
            #print(f"DEBUG: removing {path}")
            rmtree(path)
        except FileNotFoundError as e:
            pass

    #print(f"\n=== Completed input processing cycle ===\n")
    periodic.enter(INTERVAL, 1, main, (periodic,))

    return None

if __name__ == '__main__':
    if os.getenv('GPS_DEBUG', '').lower() in ['1', 'y', 'on', 'yes', 'true']:
        set_debug_aws()
    periodic = sched.scheduler(time.time, time.sleep)
    periodic.enter(INTERVAL, 1, main, (periodic,))
    periodic.run()