import time
import logging
import os
import numpy as np

logger = logging.getLogger(__name__)

class TripCubeWriter:
    def __init__(self, filename, options):
        self.output_corename = filename

    def write(self, detector):
        import getpass
        from pymchelper.shieldhit.detector.detector_type import SHDetType
        from pymchelper import __version__ as _pmcversion
        # TODO add printing information how to install pytrip if it's missing
        from pytrip import __version__ as _ptversion

        pixel_size_x = (detector.xmax - detector.xmin) / detector.nx
        pixel_size_z = (detector.zmax - detector.zmin) / detector.nz

        logging.debug("psx: {:.6f} [cm]".format(pixel_size_x))
        logging.debug("psz: {:.6f} [cm]".format(pixel_size_z))

        _patient_name = "Anonymous"
        _created_by = getpass.getuser()
        _creation_info = "Created with pymchelper {:s}; using PyTRiP98 {:s}".format(_pmcversion,
                                                                                _ptversion)

        if detector.dettyp == SHDetType.dose:

            from pytrip import dos

            cube = dos.DosCube()
            # Warning: PyTRiP cube dimensions are in [mm]
            cube.create_empty_cube(
                1.0, detector.nx, detector.ny, detector.nz,
                pixel_size=pixel_size_x * 10.0,
                slice_distance=pixel_size_z * 10.0)

            # .dos dose cubes are usually in normalized integers,
            # where "1000" equals 100.0 % dose.
            # The next are also the defaults, but just to be clear
            # this is specifially set.
            cube.data_type = "integer"
            cube.num_bytes = 2
            cube.pydata_type = np.int16

            cube.cube = detector.data.reshape(detector.nx, detector.ny, detector.nz)

            if detector.tripdose >= 0.0 and detector.tripntot > 0:
                cube.cube = (cube.cube * detector.tripntot * 1.602e-10) / detector.tripdose * 1000.0
            else:
                cube.cube = (cube.cube / cube.cube.max()) * 1200.0

            # Save proper meta information
            cube.patient_name = _patient_name
            cube.created_by = _created_by
            cube.creation_info = _creation_info

            cube.write(self.output_corename)

        elif detector.dettyp in (SHDetType.dlet, SHDetType.tlet, SHDetType.dletg, SHDetType.tletg):

            from pytrip import let

            cube = let.LETCube()
            # Warning: PyTRiP cube dimensions are in [mm]
            cube.create_empty_cube(
                1.0, detector.nx, detector.ny, detector.nz,
                pixel_size=pixel_size_x * 10.0,
                slice_distance=pixel_size_z * 10.0)

            # .dosemlet.dos LET cubes are usually in 32 bit floats.
            cube.data_type = "float"
            cube.num_bytes = 4
            cube.pydata_type = np.float32

            # need to redo the cube, since by default np.float32 are allocated.
            # When https://github.com/pytrip/pytrip/issues/35 is fixed,
            # then this should not be needed.
            cube.cube = np.ones((cube.dimz, cube.dimy, cube.dimx), dtype=cube.pydata_type)

            cube.cube = detector.data.reshape(detector.nx, detector.ny, detector.nz)
            cube.cube *= 0.1  # MeV/cm -> keV/um
            # Save proper meta information

            cube.patient_name = _patient_name
            cube.created_by = _created_by
            cube.creation_info = _creation_info

            cube.write(self.output_corename)
        else:
            logger.error("Tripcube target is only allowed with dose- or LET-type detectors.")
            raise Exception("Illegal detector for tripcube.")

class TripDddWriter(object):

    _ddd_header_template = """!filetype    ddd
!fileversion   {fileversion:s}
!filedate      {filedate:s}
!projectile    {projectile:s}
!material      {material:s}
!composition   {composition:s}
!density {density:f}
!energy {energy:f}
#   z[g/cm**2] dE/dz[MeV/(g/cm**2)] FWHM1[g/cm**2] factor FWHM2[g/cm**2]
!ddd
"""

    def __init__(self, filename, options):

        import matplotlib
        matplotlib.use('Agg')
        self.ddd_filename = filename
        self.energy_MeV = options.energy
        self.ngauss = options.ngauss
        self.verbosity = options.verbose
        if not self.ddd_filename.endswith(".ddd"):
            self.ddd_filename += ".ddd"
        self.outputdir = os.path.abspath(os.path.dirname(self.ddd_filename))

    def write(self, detector):
        from pymchelper.shieldhit.detector.detector_type import SHDetType

        if detector.dettyp == SHDetType.ddd:

            # extract data from detector data
            self._extract_data(detector)

            # in order to avoid fitting data to noisy region far behind Bragg peak tail,
            # find the range of z coordinate which containes (1-threshold) of the deposited energy
            threshold = 3e-3
            cum_dose = self._cumulative_dose()
            cum_dose_left = self._cumulative_dose_left(cum_dose)

            thr_ind = cum_dose_left.size - np.searchsorted(cum_dose_left[::-1], threshold) - 1
            z_fitting_cm_1d = self.z_data_cm_1d[:thr_ind]
            dose_fitting_MeV_g_1d = self.dose_data_MeV_g_1d[:thr_ind]

            r_fitting_cm_2d, z_fitting_cm_2d = np.meshgrid(self.r_data_cm_1d, z_fitting_cm_1d)
            dose_fitting_MeV_g_2d = self.dose_data_MeV_g_2d[0:thr_ind]

            logger.info("Plotting 1..")
            if self.verbosity > 0:
                self._pre_fitting_plots(
                    cum_dose_left=cum_dose_left,
                    z_fitting_cm_1d=z_fitting_cm_1d,
                    dose_fitting_MeV_g_1d=dose_fitting_MeV_g_1d,
                    threshold=threshold,
                    zmax_cm=z_fitting_cm_1d[-1])

                self._plot_2d_map(z_fitting_cm_2d, r_fitting_cm_2d, dose_fitting_MeV_g_2d, z_fitting_cm_1d)

            logger.info("Fitting...")
            fwhm1_cm_data = np.zeros_like(z_fitting_cm_1d)
            fwhm2_cm_data = np.zeros_like(z_fitting_cm_1d)
            weight_data = np.zeros_like(z_fitting_cm_1d)
            dz0_MeV_cm_g_data = np.zeros_like(z_fitting_cm_1d)
            fwhm1_cm_error_data = np.zeros_like(z_fitting_cm_1d)
            fwhm2_cm_error_data = np.zeros_like(z_fitting_cm_1d)
            weight_error_data = np.zeros_like(z_fitting_cm_1d)
            dz0_MeV_cm_g_error_data = np.zeros_like(z_fitting_cm_1d)
            if self.ngauss in (1, 2):
                # for each depth fit a lateral beam with gaussian models
                for ind, z_cm in enumerate(z_fitting_cm_1d):

                    dose_at_z = self.dose_data_MeV_g_2d[ind]

                    # take into account only this position in r for which dose is positive
                    r_fitting_cm = self.r_data_cm_1d[dose_at_z > 0]
                    dose_fitting_1d_positive_MeV_g = dose_at_z[dose_at_z > 0]

                    # perform the fit
                    params, params_error = self._lateral_fit(r_fitting_cm,
                                                             dose_fitting_1d_positive_MeV_g,
                                                             z_cm,
                                                             self.energy_MeV,
                                                             self.ngauss)

                    fwhm1_cm, factor, fwhm2_cm, dz0_MeV_cm_g = params
                    fwhm1_cm_error, factor_error, fwhm2_cm_error, dz0_MeV_cm_g_error = params_error
                    fwhm1_cm_data[ind] = fwhm1_cm
                    dz0_MeV_cm_g_data[ind] = dz0_MeV_cm_g
                    fwhm1_cm_error_data[ind] = fwhm1_cm_error
                    dz0_MeV_cm_g_error_data[ind] = dz0_MeV_cm_g_error
                    if self.ngauss == 2:
                        fwhm2_cm_data[ind] = fwhm2_cm  # set to 0 in case ngauss = 1
                        weight_data[ind] = factor  # set to 0 in case ngauss = 1
                        fwhm2_cm_error_data[ind] = fwhm2_cm_error
                        weight_error_data[ind] = factor_error

            logger.info("Plotting 2...")
            if self.verbosity > 0 and self.ngauss in (1, 2):
                self._post_fitting_plots(z_fitting_cm_1d,
                                         dose_fitting_MeV_g_1d,
                                         dz0_MeV_cm_g_data,
                                         fwhm1_cm_data,
                                         fwhm2_cm_data,
                                         weight_data,
                                         dz0_MeV_cm_g_error_data,
                                         fwhm1_cm_error_data,
                                         weight_error_data,
                                         fwhm2_cm_error_data)
                self._plot_2d_map(
                    z_fitting_cm_2d,
                    r_fitting_cm_2d,
                    dose_fitting_MeV_g_2d,
                    z_fitting_cm_1d,
                    fwhm1_cm_data,
                    fwhm2_cm_data,
                    weight_data,
                    dz0_MeV_cm_g_data,
                    suffix='_fwhm')

            logger.info("Writing " + self.ddd_filename)

            # prepare header of DDD file
            header = self._ddd_header_template.format(
                fileversion='19980520',
                filedate=time.strftime('%c'),  # Locale's appropriate date and time representation
                projectile='C',
                material='H20',
                composition='H20',
                density=1,
                energy=self.energy_MeV)

            # write the contents of the files
            with open(self.ddd_filename, 'w') as ddd_file:
                ddd_file.write(header)
                # TODO write to DDD gaussian amplitude, not the dose in central bin
                if self.ngauss == 2:
                    for z_cm, dose, fwhm1_cm, weight, fwhm2_cm in zip(z_fitting_cm_1d, dose_fitting_MeV_g_1d,
                                                                      fwhm1_cm_data, weight_data, fwhm2_cm_data):
                        ddd_file.write('{:g} {:g} {:g} {:g} {:g}\n'.format(z_cm, dose, fwhm1_cm, weight, fwhm2_cm))
                elif self.ngauss == 1:
                    for z_cm, dose, fwhm_cm in zip(z_fitting_cm_1d, dose_fitting_MeV_g_1d, fwhm1_cm_data):
                        ddd_file.write('{:g} {:g} {:g}\n'.format(z_cm, dose, fwhm_cm))
                elif self.ngauss == 0:
                    for z_cm, dose in zip(z_fitting_cm_1d, dose_fitting_MeV_g_1d):
                        ddd_file.write('{:g} {:g}\n'.format(z_cm, dose))

    def _extract_data(self, detector):
        # 2D arrays of r,z and dose
        self.r_data_cm_2d = np.array(list(detector.x)).reshape(detector.nz, detector.nx)
        self.z_data_cm_2d = np.array(list(detector.z)).reshape(detector.nz, detector.nx)
        self.dose_data_MeV_g_2d = np.array(detector.v).reshape(detector.nz, detector.nx)

        self.dose_error_MeV_g_2d = np.array(detector.error).reshape(detector.nz, detector.nx)

        # 1D arrays of r,z and dose in the very central bin
        self.r_data_cm_1d = self.r_data_cm_2d[0]  # middle points of the bins
        self.z_data_cm_1d = np.asarray(list(detector.z)[0:detector.nz * detector.nx:detector.nx])

        # np.savez("data", r2d=self.r_data_cm_2d, z2d=self.z_data_cm_2d, d2d=self.dose_data_MeV_g_2d,
        #          r1d=self.r_data_cm_1d, z1d=self.z_data_cm_1d, e2d=self.dose_error_MeV_g_2d)

        bin_depth_z_cm = self.z_data_cm_1d[1] - self.z_data_cm_1d[0]
        r_step_cm = self.r_data_cm_1d[1] - self.r_data_cm_1d[0]

        # i-th bin volume = dz * pi * (r_i_max^2 - r_i_min^2  )
        #   r_i_max = r_i + dr / 2
        #   r_i_min = r_i - dr / 2
        #  r_i_max^2 - r_i_min^2 = (r_i_max - r_i_min)*(r_i_max + r_i_min) = dr * 2 * r_i    thus
        # i-th bin volume = 2 * pi * dr * r_i * dz
        bin_volume_data_cm3_1d = 2.0 * np.pi * r_step_cm * self.r_data_cm_1d * bin_depth_z_cm
        # we assume density of 1 g/c3
        density_g_cm3 = 1.0
        total_bin_mass_g = density_g_cm3 * bin_depth_z_cm * np.pi * (self.r_data_cm_1d[-1] + r_step_cm/2.0)**2
        energy_in_bin_MeV_2d = self.dose_data_MeV_g_2d * bin_volume_data_cm3_1d * density_g_cm3
        total_energy_at_depth_MeV_1d = np.sum(energy_in_bin_MeV_2d, axis=1)
        self.dose_data_MeV_g_1d = total_energy_at_depth_MeV_1d / total_bin_mass_g

    def _cumulative_dose(self):
        cumsum = np.cumsum(self.dose_data_MeV_g_1d)
        cumsum /= np.sum(self.dose_data_MeV_g_1d)
        return cumsum

    def _cumulative_dose_left(self, cumsum):
        cum_dose_left = np.array(cumsum)
        cum_dose_left *= -1.0
        cum_dose_left += 1.0
        return cum_dose_left

    def _plot_2d_map(self,
                     z_fitting_cm_2d,
                     r_fitting_cm_2d,
                     dose_fitting_MeV_g2d,
                     z_fitting_cm_1d=None,
                     fwhm1_cm=None,
                     fwhm2_cm=None,
                     weight=None,
                     dz0_MeV_cm_g_data=None,
                     suffix=''):
        import matplotlib
        matplotlib.use('Agg')
        import matplotlib.pyplot as plt
        from matplotlib.colors import LogNorm

        prefix = os.path.join(self.outputdir, 'ddd_{:3.1f}MeV_'.format(self.energy_MeV))
        plt.pcolormesh(
            z_fitting_cm_2d, r_fitting_cm_2d, dose_fitting_MeV_g2d, norm=LogNorm(), cmap='gnuplot2', label='dose')
        cbar = plt.colorbar()
        cbar.set_label("dose [MeV/g]", rotation=270, verticalalignment='bottom')
        if z_fitting_cm_1d is not None and np.any(fwhm1_cm):
            plt.plot(z_fitting_cm_1d, fwhm1_cm, color='g', label="fwhm1")
        if z_fitting_cm_1d is not None and np.any(fwhm2_cm):
            plt.plot(z_fitting_cm_1d, fwhm2_cm, color='r', label="fwhm2")

        # plot legend only if some of the FWHM 1-D overlays are present
        # adding legend to only pcolormesh plot will result in a warning about missing labels
        if z_fitting_cm_1d is not None and (np.any(fwhm1_cm) or np.any(fwhm2_cm)):
            plt.legend(loc=0)
        plt.xlabel("z [cm]")
        plt.ylabel("r [cm]")
        plt.xlim((z_fitting_cm_2d.min(), z_fitting_cm_2d.max()))
        if np.any(fwhm1_cm) and np.any(fwhm2_cm):
            plt.ylim((r_fitting_cm_2d.min(), max(max(fwhm1_cm), max(fwhm2_cm))))
        plt.clim((1e-8 * dose_fitting_MeV_g2d.max(), dose_fitting_MeV_g2d.max()))
        out_filename = prefix + 'dosemap' + suffix + '.png'
        logger.info('Saving ' + out_filename)
        plt.savefig(out_filename)
        if self.verbosity > 1:
            plt.yscale('log')
            out_filename = prefix + 'dosemap_log' + suffix + '.png'
            logger.info('Saving ' + out_filename)
            plt.savefig(out_filename)
        plt.close()

        if self.verbosity > 2 and (np.any(fwhm1_cm) or np.any(fwhm2_cm)):
            # TODO add plotting sum of 2 gausses
            sigma1_cm = fwhm1_cm / 2.354820045
            sigma2_cm = fwhm2_cm / 2.354820045
            gauss_amplitude_MeV_g = dz0_MeV_cm_g_data
            for z_cm, sigma1_at_z_cm, sigma2_at_z_cm, factor, amplitude_MeV_g in \
                    zip(z_fitting_cm_1d, sigma1_cm, sigma2_cm, weight, gauss_amplitude_MeV_g):
                dose_mc_MeV_g = self.dose_data_MeV_g_2d[self.z_data_cm_2d == z_cm]
                title = "Z = {:4.3f} cm,  sigma1 = {:4.3f} cm".format(z_cm, sigma1_at_z_cm)
                plt.plot(self.r_data_cm_1d, dose_mc_MeV_g, 'k.', label="data")
                if self.ngauss == 1:
                    gauss_data_MeV_g = self.gauss_MeV_g(self.r_data_cm_1d, amplitude_MeV_g, sigma1_at_z_cm)
                    plt.plot(self.r_data_cm_1d, gauss_data_MeV_g, label="fit")
                elif self.ngauss == 2:
                    gauss_data_MeV_g = self.gauss2_MeV_g(self.r_data_cm_1d, amplitude_MeV_g,
                                                         sigma1_at_z_cm, factor, sigma2_at_z_cm)
                    gauss_data_MeV_g_1st = self.gauss2_MeV_g_1st(self.r_data_cm_1d, amplitude_MeV_g,
                                                                 sigma1_at_z_cm, factor, sigma2_at_z_cm)
                    gauss_data_MeV_g_2nd = self.gauss2_MeV_g_2nd(self.r_data_cm_1d, amplitude_MeV_g,
                                                                 sigma1_at_z_cm, factor, sigma2_at_z_cm)
                    plt.plot(self.r_data_cm_1d, gauss_data_MeV_g, label="fit")
                    plt.plot(self.r_data_cm_1d, gauss_data_MeV_g_1st, label="fit 1st gauss")
                    plt.plot(self.r_data_cm_1d, gauss_data_MeV_g_2nd, label="fit 2nd gauss")
                    title += ", sigma2 = {:4.3f} cm, factor = {:4.6f}".format(sigma2_at_z_cm, factor)
                logger.debug("Plotting at " + title)
                plt.title(title)
                plt.legend(loc=0)
                plt.yscale('log')
                plt.xlabel("r [cm]")
                plt.ylabel("dose [MeV/g]")
                plt.ylim([dose_mc_MeV_g.min(), dose_mc_MeV_g.max()])
                if self.ngauss == 1:
                    plt.ylim([dose_mc_MeV_g.min(), max(gauss_data_MeV_g.max(), dose_mc_MeV_g.max())])
                out_filename = prefix + "fit_details_{:4.3f}_log".format(z_cm) + suffix + '.png'
                logger.info('Saving ' + out_filename)
                plt.savefig(out_filename)

                plt.xscale('log')
                plt.xlim([0, self.r_data_cm_1d.max()])
                out_filename = prefix + "fit_details_{:4.3f}_loglog".format(z_cm) + suffix + '.png'
                logger.info('Saving ' + out_filename)
                plt.savefig(out_filename)

                plt.xscale('linear')
                plt.xlim([0, 5.0*sigma2_at_z_cm])
                plt.ylim([dose_mc_MeV_g[self.r_data_cm_1d < 5.0*sigma2_at_z_cm].min(), dose_mc_MeV_g.max()])
                out_filename = prefix + "fit_details_{:4.3f}_small_log".format(z_cm) + suffix + '.png'
                logger.info('Saving ' + out_filename)
                plt.savefig(out_filename)

                plt.close()

                plt.plot(self.r_data_cm_1d, dose_mc_MeV_g * self.r_data_cm_1d, 'k.', label="data")
                plt.plot(self.r_data_cm_1d, gauss_data_MeV_g * self.r_data_cm_1d, label="fit")
                plt.legend(loc=0)
                plt.ylabel("dose * r [MeV cm/g]")
                plt.ylim([(dose_mc_MeV_g * self.r_data_cm_1d).min(), (dose_mc_MeV_g * self.r_data_cm_1d).max()])
                plt.yscale('log')
                out_filename = prefix + "fit_details_{:4.3f}_r_log".format(z_cm) + suffix + '.png'
                logger.info('Saving ' + out_filename)
                plt.savefig(out_filename)
                plt.xscale('log')
                out_filename = prefix + "fit_details_{:4.3f}_r_loglog".format(z_cm) + suffix + '.png'
                logger.info('Saving ' + out_filename)
                plt.savefig(out_filename)
                plt.close()

    def _pre_fitting_plots(self, cum_dose_left, z_fitting_cm_1d, dose_fitting_MeV_g_1d, threshold, zmax_cm):
        import matplotlib
        matplotlib.use('Agg')
        import matplotlib.pyplot as plt
        prefix = os.path.join(self.outputdir, 'ddd_{:3.1f}MeV_'.format(self.energy_MeV))

        plt.plot(self.z_data_cm_1d, self.dose_data_MeV_g_1d, color='blue', label='dose')
        plt.axvspan(
            0,
            zmax_cm,
            alpha=0.1,
            color='green',
            label="fitting area, covers {:g} % of dose".format(100.0 * (1 - threshold)))
        plt.legend(loc=0)
        plt.xlabel('z [cm]')
        plt.ylabel('dose [a.u.]')
        if self.verbosity > 1:
            out_filename = prefix + 'dose_all.png'
            logger.info('Saving ' + out_filename)
            plt.savefig(out_filename)
        plt.yscale('log')
        out_filename = prefix + 'all_log.png'
        logger.info('Saving ' + out_filename)
        plt.savefig(out_filename)
        plt.close()

        if self.verbosity > 1:
            bp_max_z_pos_cm = self.z_data_cm_1d[self.dose_data_MeV_g_1d == self.dose_data_MeV_g_1d.max()]

            plt.semilogy(self.z_data_cm_1d, cum_dose_left, color='blue', label="cumulative missing dose")
            plt.axvspan(
                0,
                zmax_cm,
                alpha=0.1,
                color='green',
                label="fitting area, covers {:g} % of dose".format(100.0 * (1 - threshold)))
            plt.axhline(threshold, color='black', label='threshold {:g}'.format(threshold))
            plt.axvline(bp_max_z_pos_cm, color='red', label='BP max')
            plt.legend(loc=0)
            plt.xlabel('z [cm]')
            plt.ylabel('fraction of total dose deposited behind z')
            out_filename = prefix + 'dose_frac.png'
            logger.info('Saving ' + out_filename)
            plt.savefig(out_filename)
            plt.close()

        if self.verbosity > 1:
            plt.plot(z_fitting_cm_1d, dose_fitting_MeV_g_1d, 'b', label='dose')
            plt.xlabel('z [cm]')
            plt.ylabel('dose [MeV/g]')
            plt.yscale('log')
            out_filename = prefix + 'dose_log.png'
            logger.info('Saving ' + out_filename)
            plt.savefig(out_filename)
            plt.close()

    def _post_fitting_plots(self, z_fitting_cm_1d,
                            dose_fitting_MeV_g_1d,
                            dz0_MeV_cm_g_data,
                            fwhm1_cm_data,
                            fwhm2_cm_data,
                            weight_data,
                            dz0_MeV_cm_g_error_data,
                            fwhm1_cm_error_data,
                            weight_error_data,
                            fwhm2_cm_error_data):
        import matplotlib.pyplot as plt
        prefix = os.path.join(self.outputdir, 'ddd_{:3.1f}MeV_'.format(self.energy_MeV))

        # left Y axis dedicated to FWHM, right one to weight
        fig, ax1 = plt.subplots()
        ax2 = ax1.twinx()
        lns1 = ax1.plot(z_fitting_cm_1d, fwhm1_cm_data, 'g', label='fwhm1')
        upper_fwhm1_line = fwhm1_cm_data + fwhm1_cm_error_data
        lower_fwhm1_line = fwhm1_cm_data - fwhm1_cm_error_data
        ax1.fill_between(z_fitting_cm_1d, lower_fwhm1_line, upper_fwhm1_line,
                         where=upper_fwhm1_line >= lower_fwhm1_line,
                         facecolor='green',
                         alpha=0.1,
                         interpolate=True)
        if self.ngauss == 2:
            lns2 = ax1.plot(z_fitting_cm_1d, fwhm2_cm_data, 'r', label='fwhm2')
            upper_fwhm2_line = fwhm2_cm_data + fwhm2_cm_error_data
            lower_fwhm2_line = fwhm2_cm_data - fwhm2_cm_error_data
            ax1.fill_between(z_fitting_cm_1d, lower_fwhm2_line, upper_fwhm2_line,
                             where=upper_fwhm2_line >= lower_fwhm2_line,
                             facecolor='red',
                             alpha=0.1,
                             interpolate=True)

            lns3 = ax2.plot(z_fitting_cm_1d, weight_data, 'b', label='weight')
            upper_weight_line = weight_data + weight_error_data
            lower_weight_line = weight_data - weight_error_data
            ax2.fill_between(z_fitting_cm_1d, lower_weight_line, upper_weight_line,
                             where=upper_weight_line >= lower_weight_line,
                             facecolor='blue',
                             alpha=0.1,
                             interpolate=True)
            ax2.set_ylabel('weight of FWHM1')
            ax2.set_ylim([0, 1])
        ax1.set_xlabel('z [cm]')
        ax1.set_ylabel('FWHM [cm]')

        # add by hand line plots and labels to legend
        line_objs = lns1
        if self.ngauss == 2:
            line_objs += lns2
            line_objs += lns3
        labels = [l.get_label() for l in line_objs]
        ax1.legend(line_objs, labels, loc=0)

        out_filename = prefix + 'fwhm.png'
        logger.info('Saving ' + out_filename)
        plt.savefig(out_filename)
        plt.close()

        r_step_cm = self.r_data_cm_1d[1] - self.r_data_cm_1d[0]
        r_max_cm = self.r_data_cm_1d[-1] + 0.5 * r_step_cm

        # beam model for single gaussian is following:
        #  G(r, sigma) = 1 / (2pi sigma) * exp( - 0.5 r^2 / sigma^2)
        #  D(z,r) = D(z,0) * G(r, sigma)
        # for double gaussian it is following:
        #  D(z,r) = D(z,0) * ( w * G(r, sigma1) + (1-w) * G(r, sigma2))
        #
        # to get depth dose profile D(z) we need to calculate average dose in some volume at depth z
        # (calculating average dose in a subspace separated by two planes at z=z0 and z=z0+dz will lead to zero dose)
        # we cannot use simple arithmetic mean, as we are dealing with cylindrical scoring and bin mass depends on r
        #
        # let rmax be radius of biggest bin in cylindrical scoring
        # we calculate D(z) which will correspond to depth-dose profile measured with ion. chamber of radius rmax
        # it is basically energy E(z) deposited in slice of radius rmax and thickness dz divided by slice mass
        # D(z) = E(z) / m(z) = E(z) / (pi rmax^2 dz rho)
        # energy E(z) is the sum of energy in all cylyndrical shell in a slice and can be calculated as integral
        # thin shell has surface at radius r has surface: 2 pi r dr, thus
        # E(z) = \int_0^rmax D(r,z) rho dz 2 pi r dr
        # finally:
        # D(z) = \int_0^rmax D(r,z) rho dz 2 pi r dr / (pi rmax^2 dz rho)   which leads to:
        #
        # D(z) = 2 / rmax^2 \int_0^rmax D(r,z) r dr
        #
        # for single gaussian model this gives:
        #
        # D(z) = 2 / rmax^2 \int_0^rmax D(z,0) * G(r, sigma) r dr = D(z,0) / rmax^2 \int_0^rmax G(r, sigma) r dr
        #      = D(z,0) / (2 pi sigma rmax^2) \int_0^rmax exp( - 0.5 r^2 / sigma^2) r dr
        #
        # integral \int exp( - 0.5 r^2 / sigma^2) r dr is easy to calculate:
        # https://www.wolframalpha.com/input/?i=%5Cint+exp(+-+0.5+r%5E2+%2F+sigma%5E2)+r+dr
        #
        #  \int exp( - 0.5 r^2 / sigma^2) r dr = -sigma^2 exp( - 0.5 r^2 / sigma^2)
        #
        # which leads to
        #
        # \int_0^rmax exp( - 0.5 r^2 / sigma^2) r dr = sigma^2 ( 1 - exp( - 0.5 rmax^2 / sigma^2))
        #
        # this means depth-dose curve for single gaussian model can be expressed as:
        #
        # D(z) = D(z,0) / (2 pi sigma rmax^2) * sigma^2 ( 1 - exp( - 0.5 rmax^2 / sigma^2))
        #
        # or
        #
        # D(z) = sigma * D(z,0) * ( 1 - exp( - 0.5 rmax^2 / sigma^2)) / (2 pi rmax^2)
        #
        # double gaussian can be calculated in similar way and leads to:
        #
        # D(z) = D(z,0) / (2 pi rmax^2) * ( w * sigma1 * ( 1 - exp( - 0.5 rmax^2 / sigma1^2)) +
        #                                 ( (1-w) * sigma2 * ( 1 - exp( - 0.5 rmax^2 / sigma2^2)))
        #

        if self.ngauss == 1:
            sigma1_cm = fwhm1_cm_data / 2.354820045
            # sigma * D(z,0) / (2 pi rmax^2)
            fit_dose_MeV_g = sigma1_cm * dz0_MeV_cm_g_data / (2.0 * np.pi * r_max_cm ** 2)
            # missing ( 1 - exp( - 0.5 rmax^2 / sigma^2))
            fit_dose_MeV_g *= (np.ones_like(sigma1_cm) - np.exp(-0.5 * r_max_cm / sigma1_cm**2))
            plt.plot(z_fitting_cm_1d, fit_dose_MeV_g, 'r', label='dose fit')
        if self.ngauss == 2:
            sigma1_cm = fwhm1_cm_data / 2.354820045
            sigma2_cm = fwhm2_cm_data / 2.354820045
            w = weight_data

            # ( w * sigma1 * ( 1 - exp( - 0.5 rmax^2 / sigma1^2))
            fit_dose_MeV_g = w * sigma1_cm * \
                (np.ones_like(sigma1_cm) - np.exp(-0.5 * r_max_cm / sigma1_cm**2))

            # ( (1-w) * sigma2 * ( 1 - exp( - 0.5 rmax^2 / sigma2^2)))
            fit_dose_MeV_g += (np.ones_like(w) - w) * sigma2_cm * \
                (np.ones_like(sigma2_cm) - np.exp(-0.5 * r_max_cm / sigma2_cm**2))

            # D(z,0) / (2 pi rmax^2)
            fit_dose_MeV_g *= dz0_MeV_cm_g_data / (2.0 * np.pi * r_max_cm ** 2)
            plt.plot(z_fitting_cm_1d, fit_dose_MeV_g, 'r', label='dose fit')

        plt.plot(z_fitting_cm_1d, dose_fitting_MeV_g_1d, 'b', label='dose MC')
        plt.xlabel('z [cm]')
        plt.ylabel('dose [MeV/g]')
        plt.legend(loc=0)
        out_filename = prefix + 'dose_fit.png'
        logger.info('Saving ' + out_filename)
        plt.savefig(out_filename)
        if self.verbosity > 1:
            plt.yscale('log')
            out_filename = prefix + 'dose_fit_log.png'
            logger.info('Saving ' + out_filename)
            plt.savefig(out_filename)
        plt.close()

    @classmethod
    def gauss_MeV_g(cls, x_cm, amp_MeV_cm_g, sigma_cm):
        return amp_MeV_cm_g / (2.0 * np.pi * sigma_cm) * np.exp(-x_cm ** 2 / (2.0 * sigma_cm ** 2))

    @classmethod
    def gauss_r_MeV_cm_g(cls, x_cm, amp_MeV_cm_g, sigma_cm):
        return cls.gauss_MeV_g(x_cm, amp_MeV_cm_g, sigma_cm) * x_cm

    @classmethod
    def gauss2_MeV_g(cls, x_cm, amp_MeV_cm_g, sigma1_cm, weight, sigma2_add_cm):
        return amp_MeV_cm_g / (2.0 * np.pi) * (
            (weight / sigma1_cm) * np.exp(-x_cm ** 2 / (2.0 * sigma1_cm ** 2))
            + ((1.0 - weight) / (sigma1_cm + sigma2_add_cm)) * np.exp(
                -x_cm ** 2 / (2.0 * (sigma1_cm + sigma2_add_cm) ** 2))
        )

    @classmethod
    def gauss2_MeV_g_1st(cls, x_cm, amp_MeV_cm_g, sigma1_cm, weight, sigma2_add_cm):
        return amp_MeV_cm_g / (2.0 * np.pi) * (weight / sigma1_cm) * np.exp(-x_cm ** 2 / (2.0 * sigma1_cm ** 2))

    @classmethod
    def gauss2_MeV_g_2nd(cls, x_cm, amp_MeV_cm_g, sigma1_cm, weight, sigma2_add_cm):
        return amp_MeV_cm_g / (2.0 * np.pi) * ((1.0 - weight) / (sigma1_cm + sigma2_add_cm)) * np.exp(
            -x_cm ** 2 / (2.0 * (sigma1_cm + sigma2_add_cm) ** 2))

    @classmethod
    def gauss2_r_MeV_cm_g(cls, x_cm, amp_MeV_cm_g, sigma1_cm, weight, sigma2_add_cm):
        return cls.gauss2_MeV_g(x_cm, amp_MeV_cm_g, sigma1_cm, weight, sigma2_add_cm) * x_cm

    @classmethod
    def _lateral_fit(cls, r_cm, dose_MeV_g, z_cm, energy_MeV, ngauss=2):
        variance = np.average(r_cm ** 2, weights=dose_MeV_g)

        starting_amp_MeV_g = dose_MeV_g.max()
        starting_sigma_cm = np.sqrt(variance)

        min_amp_MeV_g = 1e-10 * dose_MeV_g.max()
        min_sigma_cm = 1e-2 * starting_sigma_cm

        max_amp_MeV_g = 2.0 * dose_MeV_g.max()
        max_sigma_cm = 1e4 * starting_sigma_cm

        from scipy.optimize import curve_fit

        if ngauss == 1:
            popt, pcov = curve_fit(f=cls.gauss_r_MeV_cm_g,
                                   xdata=r_cm,
                                   ydata=dose_MeV_g * r_cm,
                                   p0=[starting_amp_MeV_g, starting_sigma_cm],
                                   bounds=([[min_amp_MeV_g, min_sigma_cm], [max_amp_MeV_g, max_sigma_cm]]),
                                   sigma=None)
            # TODO return also parameter errors
            perr = np.sqrt(np.diag(pcov))

            dz0_MeV_cm_g, sigma_cm = popt
            dz0_MeV_cm_g_error, sigma_cm_error = perr
            factor = 0.0
            factor_error = 0.0
            fwhm2_cm = 0.0
            fwhm2_cm_error = 0.0

        elif ngauss == 2:
            starting_weigth = 0.99
            starting_sigma2_add_cm = 0.1

            min_weigth = 0.55
            min_sigma2_add_cm = 1e-1

            max_weigth = 1.0 - 1e-12
            max_sigma2_add_cm = 20.0

            popt, pcov = curve_fit(f=cls.gauss2_r_MeV_cm_g,
                                   xdata=r_cm,
                                   ydata=dose_MeV_g * r_cm,
                                   p0=[starting_amp_MeV_g, starting_sigma_cm, starting_weigth, starting_sigma2_add_cm],
                                   bounds=([min_amp_MeV_g, min_sigma_cm, min_weigth, min_sigma2_add_cm],
                                           [max_amp_MeV_g, max_sigma_cm, max_weigth, max_sigma2_add_cm]),
                                   sigma=None)
            # TODO return also parameter errors
            perr = np.sqrt(np.diag(pcov))
            dz0_MeV_cm_g_error, sigma_cm_error, factor_error, sigma2_add_cm_error = perr

            dz0_MeV_cm_g, sigma_cm, factor, sigma2_add_cm = popt
            sigma2_cm = sigma_cm + sigma2_add_cm
            sigma2_cm_error = np.sqrt(sigma_cm_error**2 + sigma2_add_cm_error**2)
            fwhm2_cm = sigma2_cm * 2.354820045
            fwhm2_cm_error = sigma2_cm_error * 2.354820045

        fwhm1_cm = sigma_cm * 2.354820045

        fwhm1_cm_error = sigma_cm_error * 2.354820045

        params = fwhm1_cm, factor, fwhm2_cm, dz0_MeV_cm_g
        params_error = fwhm1_cm_error, factor_error, fwhm2_cm_error, dz0_MeV_cm_g_error

        return params, params_error


#!/usr/bin/env python

from pip.req import parse_requirements
from setuptools import find_packages, setup

install_reqs = parse_requirements('requirements.txt', session=False)
reqs = [str(ir.req) for ir in install_reqs]

setup(
    name='asyncexec',
    version='0.3.0',
    packages=find_packages(),
    include_package_data=True,
    install_requires=reqs,
)


import speak
import twitter
import wx
class TweetGui(wx.Frame):

	global inittext
	global id
	global edit

	def __init__(self,inittext="",i="",ed=0):
		self.edit=ed
		self.id=i
		wx.Frame.__init__(self, None, title="Tweet", size=(350,200)) # initialize the wx frame
		self.Bind(wx.EVT_CLOSE, self.OnClose)
		self.panel = wx.Panel(self)
		self.main_box = wx.BoxSizer(wx.VERTICAL)
		self.text_label = wx.StaticText(self.panel, -1, "Tweet Te&xt")
		self.text = wx.TextCtrl(self.panel, -1, "",style=wx.TE_MULTILINE|wx.TE_PROCESS_ENTER|wx.TE_DONTWRAP)
		self.main_box.Add(self.text, 0, wx.ALL, 10)
		self.text.Bind(wx.EVT_TEXT_ENTER, self.Tweet)
		self.text.Bind(wx.EVT_TEXT_MAXLEN, self.Maximum)
		self.text.Bind(wx.EVT_TEXT, self.Chars)
		self.text.AppendText(inittext)
		#self.text.SetSelection(self.text.GetLastPosition()-1,self.text.GetLastPosition())
		self.text.SetMaxLength(140)
		#self.text.SetInsertionPoint(self.text.GetLastPosition())
		self.tweet = wx.Button(self.panel, wx.ID_DEFAULT, "&Send")
		self.tweet.SetDefault()
		self.tweet.Bind(wx.EVT_BUTTON, self.Tweet)
		self.main_box.Add(self.tweet, 0, wx.ALL, 10)
		self.close = wx.Button(self.panel, wx.ID_CLOSE, "&Cancel")
		self.close.Bind(wx.EVT_BUTTON, self.OnClose)
		self.main_box.Add(self.close, 0, wx.ALL, 10)
		self.panel.Layout()
	def Maximum(self,event):
		twitter.snd.play("boundary")
	def Chars(self, event):
		length=round(len(self.text.GetValue()),0)
		percent=str(round((length/140)*100,0))
		self.SetLabel("Tweet - "+str(length).split(".")[0]+" of 140 characters ("+percent+" Percent)")
	def Tweet(self, event):
		if self.edit==1:
			twitter.Delete(self.id)
		status=twitter.Tweet(self.text.GetValue(),self.id)
		if status==True:
			self.Destroy()
	def OnClose(self, event):
		"""App close event handler"""
		self.Destroy()

class QuoteGui(wx.Frame):

	global inittext
	global id

	def __init__(self,i=""):
		self.id=i
		a=twitter.api.get_status(self.id)
		self.inittext=a.text
		wx.Frame.__init__(self, None, title="Tweet", size=(350,200)) # initialize the wx frame
		self.Bind(wx.EVT_CLOSE, self.OnClose)
		self.panel = wx.Panel(self)
		self.main_box = wx.BoxSizer(wx.VERTICAL)
		self.text_label = wx.StaticText(self.panel, -1, "Tweet Te&xt")
		self.text = wx.TextCtrl(self.panel, -1, "")
		self.main_box.Add(self.text, 0, wx.ALL, 10)
		self.text2_label = wx.StaticText(self.panel, -1, "Quoting")
		self.text2 = wx.TextCtrl(self.panel, -1, "",style=wx.TE_READONLY)
		self.main_box.Add(self.text2, 0, wx.ALL, 10)
		self.text2.SetValue(self.inittext)
		self.tweet = wx.Button(self.panel, wx.ID_DEFAULT, "&Send")
		self.tweet.Bind(wx.EVT_BUTTON, self.Tweet)
		self.main_box.Add(self.tweet, 0, wx.ALL, 10)
		self.tweet.SetDefault()
		self.close = wx.Button(self.panel, wx.ID_CLOSE, "&Cancel")
		self.close.Bind(wx.EVT_BUTTON, self.OnClose)
		self.main_box.Add(self.close, 0, wx.ALL, 10)
		self.panel.Layout()
	def EVT_TEXT_ENTER(self,event):
		speak.speak("Boing")
	def Tweet(self, event):
		twitter.Quote(status=self.id,text=self.text.GetValue())
		self.Destroy()
	def OnClose(self, event):
		"""App close event handler"""
		self.Destroy()

class DMGui(wx.Frame):

	global user
	def __init__(self,i=""):
		self.user=i

		wx.Frame.__init__(self, None, title="Message", size=(350,200)) # initialize the wx frame
		self.Bind(wx.EVT_CLOSE, self.OnClose)
		self.panel = wx.Panel(self)
		self.main_box = wx.BoxSizer(wx.VERTICAL)
		self.text_label = wx.StaticText(self.panel, -1, "Tweet Te&xt")
		self.text = wx.TextCtrl(self.panel, -1, "")
		self.main_box.Add(self.text, 0, wx.ALL, 10)
		self.recip_label = wx.StaticText(self.panel, -1, "&Recipient")
		self.recip = wx.TextCtrl(self.panel, -1, "")
		self.main_box.Add(self.recip, 0, wx.ALL, 10)
		self.recip.SetValue(self.user)
		self.tweet = wx.Button(self.panel, wx.ID_DEFAULT, "&Send")
		self.tweet.Bind(wx.EVT_BUTTON, self.Tweet)
		self.main_box.Add(self.tweet, 0, wx.ALL, 10)
		self.tweet.SetDefault()
		self.close = wx.Button(self.panel, wx.ID_CLOSE, "&Cancel")
		self.close.Bind(wx.EVT_BUTTON, self.OnClose)
		self.main_box.Add(self.close, 0, wx.ALL, 10)
		self.panel.Layout()
	def Tweet(self, event):
		status=twitter.Tweet("d @"+self.recip.GetValue()+" "+self.text.GetValue(),0)
		if status==True:
			self.Destroy()
	def OnClose(self, event):
		"""App close event handler"""
		self.Destroy()

class ViewGui(wx.Frame):

	def __init__(self,text):

		wx.Frame.__init__(self, None, title="View", size=(350,200)) # initialize the wx frame
		self.Bind(wx.EVT_CLOSE, self.OnClose)
		self.panel = wx.Panel(self)
		self.main_box = wx.BoxSizer(wx.VERTICAL)
		self.text_label = wx.StaticText(self.panel, -1, "Tweet Te&xt",style=wx.TE_READONLY|wx.TE_MULTILINE)
		self.text = wx.TextCtrl(self.panel, -1, "")
		self.main_box.Add(self.text, 0, wx.ALL, 10)
		self.text.SetValue(text)
		self.close = wx.Button(self.panel, wx.ID_CLOSE, "&Close")
		self.close.Bind(wx.EVT_BUTTON, self.OnClose)
		self.main_box.Add(self.close, 0, wx.ALL, 10)
		self.panel.Layout()
	def OnClose(self, event):
		"""App close event handler"""
		self.Destroy()


import json, requests
from pathlib import Path
from urllib.request import urlretrieve
from multiprocessing import Pool

def download_channel(channel, offset=0, firsturl=False):
    Path(str(Path()) + "/" + channel).mkdir(exist_ok=True)

    while 1:
        url = f'https://focusmusic.fm/api/tracks.php?offset={offset}&channel={channel}'
        file_url = json.loads(requests.get(url, timeout=10).content).get("url").replace("\\","")
        if file_url == firsturl: break
        filepath, firsturl = str(Path()) + "/" + channel + "/" + Path(file_url).name, firsturl or file_url
        if not Path(filepath).exists():
            print(f"getting offset {offset} filename {Path(filepath).name}");
            urlretrieve(file_url, filepath)
        offset +=1

Pool().map(download_channel, ["electronic", "downtempo", "classical", "rain"])


import pandas as pd
import numpy as np

from random import randint
from sklearn.preprocessing import StandardScaler

class TmpPreprocessor(object):
	def __init__(self, number_of_samples, input_trace):
		super(TmpPreprocessor, self).__init__()
		self.number_of_samples = number_of_samples
		self.input_trace = input_trace

	def fit_transform(self, X, y, noise=False):

		# Apply Noise
		if noise==True:
			X = self.apply_noise(X)

		# Standardize Features and transform into numpy array
		X = self.standardize_features(X)

		# Rescale Labels and transform into numpy array
		y = self.rescale_labels(y)

		# Generate Sequences
		X, y = self.generate_data(X, y)

		# Train, test, validation split
		X_train, y_train, X_test, y_test, X_val, y_val = self.train_test_validation_split(X, y)

		return X_train, y_train, X_test, y_test, X_val, y_val

	def standardize_features(self, features):
		sc = StandardScaler().fit(features)
		features = sc.transform(features)

		return features

	def rescale_labels(self, labels):

		labels['x'] = labels['x'].apply(lambda x: x + 600)
		# labels['y'] = labels['y'].apply(lambda x: x - img_height)
		labels['z'] = labels['z'].apply(lambda x: x + 450)

		return labels.values

	def generate_data(self, features, labels):
		data_x = []
		data_y = []
		for i in range(0, len(features)-25):
			initial_idx = randint(0, len(features)-self.input_trace-1)
			x = features[initial_idx:initial_idx+self.input_trace,:]
			y = labels[initial_idx+int(self.input_trace/2):initial_idx+int(self.input_trace/2)+1,:]
			data_x.append(x)
			data_y.append(y)
		data_x = np.asarray(data_x)
		data_y = np.asarray(data_y)

		return data_x, data_y

	def train_test_validation_split(self, features, labels, val_samples=100, test_samples=200):

		X_test = features[0:test_samples]
		y_test = labels[0:test_samples]

		X_val = features[test_samples:test_samples + val_samples]
		y_val = labels[test_samples:test_samples + val_samples]

		X_train = features[test_samples + val_samples:]
		y_train = labels[test_samples + val_samples:]

		return X_train, y_train, X_test, y_test, X_val, y_val

	def apply_noise(self, features, mean=0.0, std_dev_size=2.5, std_dev_x_y=3.0):

		# To the x
		noise = np.random.normal(mean, std_dev_x_y, len(features))
		features["relative_x"] = features["relative_x"] + noise

		# To the y
		noise = np.random.normal(mean, std_dev_x_y, len(features))
		features["relative_y"] = features["relative_y"] + noise

		# To the size
		noise = np.random.normal(mean, std_dev_size, len(features))
		# Clip the noise
		noise[noise > std_dev_size] = std_dev_size
		noise[noise < -std_dev_size] = -std_dev_size
		features["size"] = features["size"] + noise

		return features


from django.shortcuts import render, get_object_or_404, redirect
from movies.models import Movie, Person, Genre, Language, Watcher, Country, Rating, Review, Notification, MovieList
from django.contrib.auth.decorators import login_required
from django.http import HttpResponse, Http404
from django.core import files
from django.db.models import Q
import sys
import json
import requests
import datetime
import tempfile
from movies.pagination import Pagination

def process_movies(w, ms, settings):
    if settings.get('owned', False):
        w_owned = w.owned.all()
        for m in w_owned:
            pass
    if settings.get('watched', False):
        w_ratings = Rating.objects.filter(watcher=w)
        for m in w_ratings:
            pass
    if settings.get('user', False):
        u_ratings = Rating.objects.filter(watcher=settings.get('user', False))
        for m in u_ratings:
            pass
    for m in ms:
        if settings.get('owned', False):
            m.is_owned = False
            for o in w_owned:
                if o.id == m.id:
                    m.is_owned = True
                    break
        if settings.get('watched', False):
            m.is_watched = False
            m.my_rating = 0
            for o in w_ratings:
                if o.movie_id == m.id:
                    m.is_watched = True
                    m.my_rating = o.rating
                    break
        if settings.get('user', False):
            m.user_watched = False
            m.user_rating = 0
            for o in u_ratings:
                if o.movie_id == m.id:
                    m.user_watched = True
                    m.user_rating = o.rating
                    break

@login_required
def home(request):
    movies = Movie.objects.order_by('-date_added', '-id').prefetch_related('directors', 'actors', 'genres')[:10]
    movie_count = Movie.objects.all().count()
    ratings = Rating.objects.order_by('-date_added', '-id').select_related()[:10]
    rating_count = Rating.objects.all().count()
    reviews = Review.objects.order_by('-date_added', '-id').select_related()[:10]
    review_count = Review.objects.all().count()
    lists = MovieList.objects.order_by('-date_updated', '-id')[:10]
    list_count = MovieList.objects.all().count()
    process_movies(request.user.watcher, movies, {
        'watched': True,
        'owned': True,
    })
    return render(request, 'home.html', {
        'movies': movies,
        'ratings': ratings,
        'reviews': reviews,
        'lists': lists,
        'menu_home': True,
        'show_watched': True,
        'show_owned': True,
        'movie_pagination': Pagination(1, (movie_count+9)/10),
        'rating_pagination': Pagination(1, (rating_count+9)/10),
        'review_pagination': Pagination(1, (review_count+9)/10),
        'list_pagination': Pagination(1, (list_count+9)/10),
        'counter_offset': 0,
    })

@login_required
def profile(request):
    w = request.user.watcher
    ratings = Rating.objects.filter(watcher=w).order_by('-date_added', '-id').select_related()[:10]
    rating_count = Rating.objects.filter(watcher=w).count()
    reviews = Review.objects.filter(watcher=w).order_by('-date_added', '-id').select_related()[:10]
    review_count = Review.objects.filter(watcher=w).count()
    return render(request, 'profile/profile.html', {
        'ratings': ratings,
        'reviews': reviews,
        'menu_profile': True,
        'rating_pagination': Pagination(1, (rating_count+9)/10),
        'review_pagination': Pagination(1, (review_count+9)/10),
        'counter_offset': 0,
        'hide_user': True,
    })

@login_required
def profile_waiting(request):
    w = request.user.watcher
    ms = Movie.objects.filter(owners=w).exclude(watched_by=w).order_by('year').prefetch_related('directors', 'actors', 'genres')
    process_movies(w, ms, {'watched': True})
    return render(request, 'profile/index.html', {
        'movies': ms,
        'menu_profile': True,
        'submenu_waiting': True,
        'show_watched': True,
    })

@login_required
def profile_owned(request):
    ms = request.user.watcher.owned.all().order_by('year').prefetch_related('directors', 'actors', 'genres')
    process_movies(request.user.watcher, ms, {'owned': True})
    return render(request, 'profile/index.html', {
        'movies': ms,
        'menu_profile': True,
        'submenu_owned': True,
        'show_owned': True,
    })

@login_required
def profile_watched(request):
    ms = request.user.watcher.watched.all().order_by('year').prefetch_related('directors', 'actors', 'genres')
    process_movies(request.user.watcher, ms, {'watched': True})
    return render(request, 'profile/index.html', {
        'movies': ms,
        'menu_profile': True,
        'submenu_watched': True,
        'show_watched': True,
    })

@login_required
def profile_lists(request):
    ls = MovieList.objects.filter(watcher=request.user.watcher).order_by('-date_updated')
    return render(request, 'profile/lists.html', {
        'lists': ls,
        'menu_profile': True,
        'submenu_lists': True,
    })

@login_required
def profile_settings(request):
    return render(request, 'profile/settings.html', {
        'menu_profile': True,
        'submenu_settings': True,
    })

@login_required
def profile_notifications(request):
    ns = Notification.objects.filter(watcher=request.user.watcher).order_by('-date_added')
    rs = Notification.objects.filter(target=request.user.watcher).order_by('-date_added')
    return render(request, 'profile/notifications.html', {
        'all_notifications': ns,
        'reverse_notifications': rs,
        'menu_profile': True,
        'submenu_notifications': True,
    })

@login_required
def users(request):
    ws = Watcher.objects.exclude(id=request.user.watcher.id)
    return render(request, 'users/index.html', {
        'watchers': ws,
        'menu_users': True,
        'hide_submenu': True,
    })

@login_required
def user(request, id):
    w = get_object_or_404(Watcher, id=id)
    if w == request.user.watcher:
        return redirect(profile)
    ratings = Rating.objects.filter(watcher=w).order_by('-date_added', '-id').select_related()[:10]
    rating_count = Rating.objects.filter(watcher=w).count()
    reviews = Review.objects.filter(watcher=w).order_by('-date_added', '-id').select_related()[:10]
    review_count = Review.objects.filter(watcher=w).count()
    return render(request, 'users/profile.html', {
        'ratings': ratings,
        'reviews': reviews,
        'menu_users': True,
        'menu_users_value': w.get_name(),
        'watcher': w,
        'rating_pagination': Pagination(1, (rating_count+9)/10),
        'review_pagination': Pagination(1, (review_count+9)/10),
        'counter_offset': 0,
        'hide_user': True,
    })

@login_required
def user_borrow(request, id):
    w = get_object_or_404(Watcher, id=id)
    if w==request.user.watcher:
        return redirect(profile)
    ms = Movie.objects.filter(owners=w).exclude(owners=request.user.watcher).prefetch_related('directors', 'actors', 'genres')
    process_movies(request.user.watcher, ms, {
        'owned': True,
        'user': w,
    })
    return render(request, 'users/user.html', {
        'movies': ms,
        'menu_users': True,
        'submenu_borrow': True,
        'menu_users_value': w.get_name(),
        'watcher': w,
        'show_rated': True,
        'show_owned': True,
    });

@login_required
def user_owned(request, id):
    w = get_object_or_404(Watcher, id=id)
    if w==request.user.watcher:
        return redirect(profile)
    ms = w.owned.all().order_by('year').prefetch_related('directors', 'actors', 'genres')
    process_movies(request.user.watcher, ms, {'watched': True})
    return render(request, 'users/user.html', {
        'movies': ms,
        'menu_users': True,
        'submenu_owned': True,
        'menu_users_value': w.get_name(),
        'watcher': w,
        'show_watched': True,
    })

@login_required
def user_watched(request, id):
    w = get_object_or_404(Watcher, id=id)
    if w==request.user.watcher:
        return redirect(profile)
    ms = w.watched.all().order_by('year').prefetch_related('directors', 'actors', 'genres')
    process_movies(request.user.watcher, ms, {
        'watched': True,
        'user': w,
    })
    return render(request, 'users/user.html', {
        'movies': ms,
        'menu_users': True,
        'submenu_watched': True,
        'menu_users_value': w.get_name(),
        'watcher': w,
        'show_rated': True,
        'show_watched': True,
    })

@login_required
def user_lists(request, id):
    w = get_object_or_404(Watcher, id=id)
    if w==request.user.watcher:
        return redirect(profile)
    ls = MovieList.objects.filter(watcher=w).order_by('-date_updated')
    return render(request, 'users/lists.html', {
        'lists': ls,
        'menu_users': True,
        'submenu_lists': True,
        'menu_users_value': w.get_name(),
        'watcher': w,
    })

@login_required
def show_list(request, id):
    l = get_object_or_404(MovieList, id=id)
    ms = l.movies.all().prefetch_related('directors', 'actors', 'genres')
    is_own = l.watcher == request.user.watcher
    if is_own:
        process_movies(request.user.watcher, ms, {'watched': True})
        return render(request, 'profile/list.html', {
            'list': l,
            'movies': ms,
            'menu_profile': True,
            'submenu_lists': True,
            'show_watched': True,
        })
    else:
        process_movies(request.user.watcher, ms, {
            'watched': True,
            'user': l.watcher,
        })
        return render(request, 'users/list.html', {
            'list': l,
            'movies': ms,
            'menu_users': True,
            'submenu_lists': True,
            'watcher': l.watcher,
            'show_watched': True,
            'show_rated': True,
        })

@login_required
def remove_list(request, id):
    l = get_object_or_404(MovieList, id=id)
    if l.watcher != request.user.watcher:
        raise Http404()
    l.delete()
    return redirect(profile_lists)

@login_required
def add_list(request):
    if request.method != "POST":
        raise Http404()
    title = request.POST['title']
    description = request.POST['description']
    if not title or not description:
        return redirect(profile_lists)
    MovieList.objects.create(title=title, description=description, watcher=request.user.watcher)
    return redirect(profile_lists)

@login_required
def show_list_for_movie(request, id):
    m = get_object_or_404(Movie, id=id)
    ls = MovieList.objects.filter(watcher=request.user.watcher)
    for l in ls:
        l.has_movie = (l.movies.filter(id=m.id).count() > 0)
    return render(request, 'partial/lists_movie.html', {
        'movie': m,
        'lists': ls,
    })

@login_required
def add_movie_to_list(request, list_id, movie_id):
    try:
        m = get_object_or_404(Movie, id=movie_id)
        l = get_object_or_404(MovieList, id=list_id)
    except:
        return HttpResponse(json.dumps({'status': False}), content_type='application/json')
    l.movies.add(m)
    l.date_updated = datetime.datetime.now()
    l.save()
    return HttpResponse(json.dumps({'status': True}), content_type='application/json')

@login_required
def remove_movie_from_list(request, list_id, movie_id):
    try:
        m = get_object_or_404(Movie, id=movie_id)
        l = get_object_or_404(MovieList, id=list_id)
    except:
        return HttpResponse(json.dumps({'status': False}), content_type='application/json')
    l.movies.remove(m)
    l.date_updated = datetime.datetime.now()
    l.save()
    return HttpResponse(json.dumps({'status': True}), content_type='application/json')

@login_required
def movie(request, id):
    m = get_object_or_404(Movie, id=id)
    if request.user.watcher.owned.filter(id=id).count() > 0:
        m.is_owned = True
    else:
        m.is_owned = False
    try:
        r = Rating.objects.get(watcher=request.user.watcher, movie=m)
        m.is_watched = True
        m.my_rating = r.rating
    except:
        m.is_watched = False
        m.my_rating = 0
    ratings = Rating.objects.filter(movie=m).exclude(watcher=request.user.watcher)
    reviews = Review.objects.filter(movie=m).exclude(watcher=request.user.watcher)
    try:
        my_review = Review.objects.get(watcher=request.user.watcher, movie=m)
    except:
        my_review = False
    rating_dict = dict([(o.watcher_id, o) for o in ratings])
    for r in reviews:
        r.rated = rating_dict.get(r.watcher_id, False)
    watchers = Watcher.objects.all().exclude(id=request.user.watcher.id)
    owners_dict = dict([(o.id, o.id) for o in m.owners.all()])
    reviewers_dict = dict([(r.watcher_id, r.watcher_id) for r in m.reviewed_by.all()])
    for w in watchers:
        w.owns = (w.id in owners_dict)
        w.reviewed = (w.id in reviewers_dict)
    ls = m.movielist_set.all().order_by('-date_updated', '-id')
    for l in ls:
        if l.watcher == request.user.watcher:
            l.is_own = True
        else:
            l.is_own = False
    return render(request, 'movies/movie.html', {
        'movie': m,
        'ratings': ratings,
        'reviews': reviews,
        'lists': ls,
        'watchers': watchers,
        'my_review': my_review,
        'menu_movies': True,
        'hide_submenu': True,
        'menu_movies_value': m.title,
    })

@login_required
def movie_review(request, id):
    m = get_object_or_404(Movie, id=id)
    if request.method == "POST":
        if not 'review' in request.POST or not request.POST['review']:
            return redirect(movie, id)
        try:
            review = Review.objects.get(watcher=request.user.watcher, movie=m)
            review.review = request.POST['review']
            review.save()
        except:
            review = Review.objects.create(watcher=request.user.watcher, movie=m, review=request.POST['review'])
        return redirect(movie, id)
    try:
        review = Review.objects.get(watcher=request.user.watcher, movie=m)
    except:
        review = False
    return render(request, 'movies/review.html', {
        'movie': m,
        'review': review,
        'menu_movies': True,
        'hide_submenu': True,
        'menu_movies_value': "Review (%s)" % (m.title),
    })

@login_required
def movies(request):
    ms = Movie.objects.all().order_by('year').prefetch_related('directors', 'actors', 'genres')
    process_movies(request.user.watcher, ms,{
        'watched': True,
        'owned': True,
    })
    return render(request, 'movies/index.html', {
        'movies': ms,
        'menu_movies': True,
        'submenu_all': True,
        'show_watched': True,
        'show_owned': True,
    })

@login_required
def years(request):
    ys = Movie.objects.all().order_by('year').values_list('year', flat=True).distinct()
    return render(request, 'movies/years.html', {
        'years': ys,
        'menu_movies': True,
        'submenu_years': True,
        'show_submenu_search': True,
    })

@login_required
def by_year(request, y):
    ms = Movie.objects.filter(year=y).order_by('year').prefetch_related('directors', 'actors', 'genres')
    process_movies(request.user.watcher, ms, {
        'watched': True,
        'owned': True,
    })
    return render(request, 'movies/index.html', {
        'movies': ms,
        'menu_movies': True,
        'submenu_years': True,
        'menu_movies_value': "By Year (%s)" % (y),
        'show_watched': True,
        'show_owned': True,
    })

@login_required
def genres(request):
    gs = Genre.objects.all().order_by('name')
    return render(request, 'movies/genres.html', {
        'genres': gs,
        'menu_movies': True,
        'submenu_genres': True,
        'show_submenu_search': True,
    })

@login_required
def by_genre(request, id):
    g = get_object_or_404(Genre, id=id)
    ms = Movie.objects.filter(genres=g).order_by('year').prefetch_related('directors', 'actors', 'genres')
    process_movies(request.user.watcher, ms, {
        'watched': True,
        'owned': True,
    })
    return render(request, 'movies/index.html', {
        'movies': ms,
        'menu_movies': True,
        'submenu_genres': True,
        'menu_movies_value': "By Genre (%s)" % (g.name),
        'show_watched': True,
        'show_owned': True,
    })

@login_required
def languages(request):
    ls = Language.objects.all().order_by('name')
    return render(request, 'movies/languages.html', {
        'languages': ls,
        'menu_movies': True,
        'submenu_languages': True,
        'show_submenu_search': True,
    })

@login_required
def by_language(request, id):
    l = get_object_or_404(Language, id=id)
    ms = Movie.objects.filter(language=l).order_by('year').prefetch_related('directors', 'actors', 'genres')
    process_movies(request.user.watcher, ms, {
        'watched': True,
        'owned': True,
    })
    return render(request, 'movies/index.html', {
        'movies': ms,
        'menu_movies': True,
        'submenu_languages': True,
        'menu_movies_value': "By Language (%s)" % (l.name),
        'show_watched': True,
        'show_owned': True,
    })

@login_required
def countries(request):
    cs = Country.objects.all().order_by('name')
    return render(request, 'movies/countries.html', {
        'countries': cs,
        'menu_movies': True,
        'submenu_countries': True,
        'show_submenu_search': True,
    })

@login_required
def by_country(request, id):
    c = get_object_or_404(Country, id=id)
    ms = Movie.objects.filter(country=c).order_by('year').prefetch_related('directors', 'actors', 'genres')
    process_movies(request.user.watcher, ms, {
        'watched': True,
        'owned': True,
    })
    return render(request, 'movies/index.html', {
        'movies': ms,
        'menu_movies': True,
        'submenu_countries': True,
        'menu_movies_value': "By Country (%s)" % (c.name),
        'show_watched': True,
        'show_owned': True,
    })

@login_required
def by_query(request, query):
    ms = Movie.objects.filter(Q(title__icontains=query) | Q(subtitle__icontains=query) | Q(imdbID=query)).order_by('year').prefetch_related('directors', 'actors', 'genres')
    ps = Person.objects.filter(name__icontains=query)
    if ms.count() == 1 and ps.count() == 0:
        return redirect(movie, ms[0].id)
    if ms.count() == 0 and ps.count() == 1:
        return redirect(by_person, ps[0].id)
    process_movies(request.user.watcher, ms, {
        'watched': True,
        'owned': True,
    })
    return render(request, 'movies/query.html', {
        'movies': ms,
        'persons': ps,
        'menu_movies': True,
        'submenu_query': True,
        'menu_movies_value': "By Query (%s)" % (query),
        'show_watched': True,
        'show_owned': True,
    })

@login_required
def persons(request):
    ps = Person.objects.all().order_by('name')
    return render(request, 'movies/persons.html', {
        'persons': ps,
        'menu_movies': True,
        'submenu_persons': True,
        'show_submenu_search': True,
    })

@login_required
def by_person(request, id):
    p = get_object_or_404(Person, id=id)
    directed = p.directed.all().order_by('year').prefetch_related('directors', 'actors', 'genres')
    written = p.written.all().order_by('year').prefetch_related('directors', 'actors', 'genres')
    acted_in = p.acted_in.all().order_by('year').prefetch_related('directors', 'actors', 'genres')
    process_movies(request.user.watcher, directed, {
        'watched': True,
        'owned': True,
    })
    process_movies(request.user.watcher, written, {
        'watched': True,
        'owned': True,
    })
    process_movies(request.user.watcher, acted_in, {
        'watched': True,
        'owned': True,
    })
    return render(request, 'movies/person.html', {
        'person': p,
        'directed': directed,
        'acted_in': acted_in,
        'written': written,
        'menu_movies': True,
        'submenu_persons': True,
        'menu_movies_value': "By Person (%s)" % (p.name),
        'show_watched': True,
        'show_owned': True,
    })

@login_required
def pagination_movie(request, page):
    if request.method != "POST":
        raise Http404()
    try:
        page = int(page)
    except:
        raise Http404()
    m_count = Movie.objects.all().count()
    if page*10-10 > m_count:
        raise Http404()
    ms = Movie.objects.order_by('-date_added', '-id').prefetch_related('directors', 'actors', 'genres')[page*10-10:page*10]
    process_movies(request.user.watcher, ms, {
        'watched': True,
        'owned': True,
    })
    return render(request, 'partial/movie_table.html', {
        'movies': ms,
        'show_watched': True,
        'show_owned': True,
        'pagination': Pagination(page, (m_count+9)/10),
        'counter_offset': page*10-10,
    })

@login_required
def pagination_rating(request, page):
    if request.method != "POST":
        raise Http404()
    try:
        page = int(page)
    except:
        raise Http404()
    r_count = Rating.objects.all().count()
    if page*10-10 > r_count:
        raise Http404()
    rs = Rating.objects.order_by('-date_added', '-id').select_related()[page*10-10:page*10]
    return render(request, 'partial/rating_table.html', {
        'ratings': rs,
        'pagination': Pagination(page, (r_count+9)/10),
        'counter_offset': page*10-10,
    })

@login_required
def pagination_review(request, page):
    if request.method != "POST":
        raise Http404()
    try:
        page = int(page)
    except:
        raise Http404()
    r_count = Review.objects.all().count()
    if page*10-10 > r_count:
        raise Http404()
    rs = Review.objects.order_by('-date_added', '-id').select_related()[page*10-10:page*10]
    return render(request, 'partial/review_table.html', {
        'reviews': rs,
        'pagination': Pagination(page, (r_count+9)/10),
        'counter_offset': page*10-10,
    })

@login_required
def pagination_list(request, page):
    if request.method != "POST":
        raise Http404()
    try:
        page = int(page)
    except:
        raise Http404()
    l_count = MovieList.objects.all().count()
    if page*10-10 > l_count:
        raise Http404()
    ls = MovieList.objects.order_by('-date_updated', '-id')[page*10-10:page*10]
    return render(request, 'partial/list_table.html', {
        'lists': ls,
        'pagination': Pagination(page, (l_count+9)/10),
        'counter_offset': page*10-10,
    })

@login_required
def pagination_user_movie(request, id, page):
    w = get_object_or_404(Watcher, id=id)
    if request.method != "POST":
        raise Http404()
    try:
        page = int(page)
    except:
        raise Http404()
    m_count = w.watched.all().count()
    if page*10-10 > m_count:
        raise Http404()
    ms = w.watched.order_by('-date_added', '-id').prefetch_related('directors', 'actors', 'genres')[page*10-10:page*10]
    process_movies(request.user.watcher, ms, {
        'watched': True,
        'owned': True,
        'user': w,
    })
    return render(request, 'partial/movie_table.html', {
        'movies': ms,
        'show_watched': True,
        'show_owned': True,
        'show_rated': True,
        'pagination': Pagination(page, (m_count+9)/10),
        'counter_offset': page*10-10,
    })

@login_required
def pagination_user_rating(request, id, page):
    w = get_object_or_404(Watcher, id=id)
    if request.method != "POST":
        raise Http404()
    try:
        page = int(page)
    except:
        raise Http404()
    r_count = Rating.objects.filter(watcher=w).count()
    if page*10-10 > r_count:
        raise Http404()
    rs = Rating.objects.filter(watcher=w).order_by('-date_added', '-id').select_related()[page*10-10:page*10]
    return render(request, 'partial/rating_table.html', {
        'ratings': rs,
        'pagination': Pagination(page, (r_count+9)/10),
        'counter_offset': page*10-10,
        'hide_user': True,
    })

@login_required
def pagination_user_review(request, id, page):
    w = get_object_or_404(Watcher, id=id)
    if request.method != "POST":
        raise Http404()
    try:
        page = int(page)
    except:
        raise Http404()
    r_count = Review.objects.filter(watcher=w).count()
    if page*10-10 > r_count:
        raise Http404()
    rs = Review.objects.filter(watcher=w).order_by('-date_added', '-id').select_related()[page*10-10:page*10]
    return render(request, 'partial/review_table.html', {
        'reviews': rs,
        'pagination': Pagination(page, (r_count+9)/10),
        'counter_offset': page*10-10,
        'hide_user': True,
    })

@login_required
def pagination_profile_movie(request, page):
    return pagination_user_movie(request, request.user.watcher.id, page)

@login_required
def pagination_profile_rating(request, page):
    return pagination_user_rating(request, request.user.watcher.id, page)

@login_required
def pagination_profile_review(request, page):
    return pagniation_user_rating(request, request.user.watcher.id, page)

@login_required
def notification_read(request, id):
    if request.method != "POST":
        raise Http404()
    notif = get_object_or_404(Notification, id=id, watcher=request.user.watcher)
    notif.read = True
    notif.save()
    return HttpResponse(json.dumps({'status': True}), content_type='application/json')

@login_required
def movie_suggest(request, movie_id, watcher_id):
    if request.method != "POST":
        raise Http404()
    movie = get_object_or_404(Movie, id=movie_id)
    watcher = get_object_or_404(Watcher, id=watcher_id)
    if watcher == request.user.watcher:
        raise Http404()
    Notification.create_suggestion(watcher, request.user.watcher, movie)
    return HttpResponse(json.dumps({'status': True}), content_type='application/json')

@login_required
def movie_ask_borrow(request, movie_id, watcher_id):
    if request.method != "POST":
        raise Http404()
    movie = get_object_or_404(Movie, id=movie_id)
    watcher = get_object_or_404(Watcher, id=watcher_id)
    if watcher == request.user.watcher:
        raise Http404()
    Notification.create_ask_borrow(watcher, request.user.watcher, movie)
    return HttpResponse(json.dumps({'status': True}), content_type='application/json')

@login_required
def movie_ask_review(request, movie_id, watcher_id):
    if request.method != "POST":
        raise Http404()
    movie = get_object_or_404(Movie, id=movie_id)
    watcher = get_object_or_404(Watcher, id=watcher_id)
    if watcher == request.user.watcher:
        raise Http404()
    Notification.create_ask_review(watcher, request.user.watcher, movie)
    return HttpResponse(json.dumps({'status': True}), content_type='application/json')

@login_required
def user_ask_suggest(request, id):
    if request.method != "POST":
        raise Http404()
    w = get_object_or_404(Watcher, id=id)
    if request.user.watcher == w:
        raise Http404()
    Notification.create_ask_suggestion(w, request.user.watcher)
    return HttpResponse(json.dumps({'status': True}), content_type='application/json')

@login_required
def profile_settings_mail(request):
    if request.method != "POST" or not 'value' in request.POST:
        raise Http404()
    receive = False
    if request.POST['value'] == 'true':
        receive = True
    request.user.watcher.settings_mail = receive
    request.user.watcher.save()
    return HttpResponse(json.dumps({'status': True}), content_type='application/json')

@login_required
def own(request, id):
    try:
        m = Movie.objects.get(id=id)
    except:
        return HttpResponse(json.dumps({'status': False}), content_type='application/json')
    request.user.watcher.owned.add(m)
    return HttpResponse(json.dumps({'status': True}), content_type='application/json')

@login_required
def unown(request, id):
    try:
        m = Movie.objects.get(id=id)
    except:
        return HttpResponse(json.dumps({'status': False}), content_type='application/json')
    request.user.watcher.owned.remove(m)
    return HttpResponse(json.dumps({'status': True}), content_type='application/json')

@login_required
def watch(request, id):
    if request.method != 'POST' or not 'rating' in request.POST:
        return HttpResponse(json.dumps({'status': False}), content_type='application/json')
    try:
        m = Movie.objects.get(id=id)
    except:
        return HttpResponse(json.dumps({'status': False}), content_type='application/json')
    rating = request.POST['rating']
    if Rating.objects.filter(watcher=request.user.watcher, movie=m).count() > 0:
        return HttpResponse(json.dumps({'status': False}), content_type='application/json')
    try:
        Rating.objects.create(watcher=request.user.watcher, movie=m, rating=rating)
    except:
        return HttpResponse(json.dumps({'status': False}), content_type='application/json')
    rating = int(rating)
    raters = Rating.objects.filter(movie=m).count()
    if raters == 0:
        m.avg_rating = rating
    else:
        m.avg_rating = (m.avg_rating*(raters-1)+rating)/raters;
    m.save()
    return HttpResponse(json.dumps({'status': True, 'rating': m.avg_rating}), content_type='application/json')

@login_required
def rate(request, id):
    if request.method != 'POST' or not 'rating' in request.POST:
        return HttpResponse(json.dumps({'status': False}), content_type='application/json')
    try:
        m = Movie.objects.get(id=id)
        ratingObj = Rating.objects.get(watcher=request.user.watcher, movie=m)
    except:
        return HttpResponse(json.dumps({'status': False}), content_type='application/json')
    rating = request.POST['rating']
    try:
        oldRating = ratingObj.rating
        ratingObj.rating = rating
        ratingObj.save()
    except:
        return HttpResponse(json.dumps({'status': False}), content_type='application/json')
    rating = int(rating)
    raters = Rating.objects.filter(movie=m).count()
    m.avg_rating = (m.avg_rating*(raters)+rating-oldRating)/raters;
    m.save()
    return HttpResponse(json.dumps({'status': True, 'rating': m.avg_rating}), content_type='application/json')

@login_required
def add(request):
    if request.method == "POST":
        title = request.POST['title']
        subtitle = request.POST.get('subtitle', None)
        yr = request.POST['year']
        genres = []
        for g in request.POST['genres'].split(','):
            g = g.strip()
            try:
                g = Genre.objects.get(name=g)
            except:
                g = Genre.objects.create(name=g)
            genres.append(g)
        runtime = request.POST['runtime']
        plot = request.POST['plot']
        languages = []
        for l in request.POST['languages'].split(','):
            l = l.strip()
            try:
                l = Language.objects.get(name=l)
            except:
                l = Language.objects.create(name=l)
            languages.append(l)
        countries = []
        for c in request.POST['country'].split(','):
            c = c.strip()
            try:
                c = Country.objects.get(name=c)
            except:
                c = Country.objects.create(name=c)
            countries.append(c)
        imdbID = request.POST['imdbID']
        if Movie.objects.filter(imdbID=imdbID).count() > 0:
            return redirect(movie, get_object_or_404(Movie, imdbID=imdbID).id)
        directors = []
        for d in request.POST['directors'].split(','):
            d = d.strip()
            try:
                d = Person.objects.get(name=d)
            except:
                d = Person.objects.create(name=d)
            directors.append(d)
        writers = []
        for w in request.POST['writers'].split(','):
            w = w.strip()
            try:
                w = Person.objects.get(name=w)
            except:
                w = Person.objects.create(name=w)
            writers.append(w)
        actors = []
        for a in request.POST['actors'].split(','):
            a = a.strip()
            try:
                a = Person.objects.get(name=a)
            except:
                a = Person.objects.create(name=a)
            actors.append(a)
        owned = 'owned' in request.POST
        m = Movie.objects.create(title=title, year=yr, runtime=runtime,
                        plot=plot, imdbID=imdbID, subtitle=subtitle)
        if 'url' in request.POST and request.POST['url']:
            url = request.POST['url']
            req = requests.get(url, stream=True)
            if req.status_code == requests.codes.ok:
                file_name = "%s.jpg" % (m.imdbID)
                lf = tempfile.NamedTemporaryFile()
                for block in req.iter_content(1024*8):
                    if not block:
                        break
                    lf.write(block)
                m.poster.save(file_name, files.File(lf))
        m.save()
        for g in genres:
            m.genres.add(g)
        for l in languages:
            m.language.add(l)
        for c in countries:
            m.country.add(c)
        for d in directors:
            m.directors.add(d)
        for w in writers:
            m.writers.add(w)
        for a in actors:
            m.actors.add(a)
        if owned:
            request.user.watcher.owned.add(m)
        return redirect(movie, m.id)
    return render(request, 'add.html', {
        'menu_add': True,
    })


#!/usr/bin/env python
import os
import sys
sys.path.append("/home/yunfeiguo/projects/kaggle_cs567/src")
import multiprocessing
import numpy as np
import pandas as pd
import matplotlib
matplotlib.use('Agg')
from pylab import *
import matplotlib.pyplot as plt
import dask.dataframe as dd
from yg_utils import *
from sklearn import linear_model
import logging
import argparse
from sklearn.cross_validation import KFold
logging.basicConfig(format='%(asctime)s %(message)s')
logging.warning('started')

#reading in training data
fn_train='/home/yunfeiguo/projects/kaggle_cs567/data/train.csv'
fn_test='/home/yunfeiguo/projects/kaggle_cs567/data/test.csv'
#fn_train='/home/yunfeiguo/projects/kaggle_cs567/data/train_100000.csv'
#fn_test='/home/yunfeiguo/projects/kaggle_cs567/data/smalltest.v2.csv'
#fn_out='/home/yunfeiguo/projects/kaggle_cs567/results/2nd_submission.csv'
trainData=pd.read_csv(fn_train,sep=',',index_col='Id')
testData=pd.read_csv(fn_test,sep=',',index_col='Id')
#trainData=dd.read_csv(fn_train,sep=',')
#trainData.set_index('Id')
#testData=dd.read_csv(fn_test,sep=',')
#testData.set_index('Id')
n=len(trainData.columns)
logging.warning('reading done')

#drop records whose Ref columns are all NaNs
trainCleaned = trainData.groupby(trainData.index).filter(lambda x: sum(np.isfinite(x['Ref'])) > 1)
#fill remaining NaNs with column mean
#alternatively, try group mean
trainCleaned.fillna(trainCleaned.mean().to_dict(),inplace=True)
testData.fillna(testData.mean().to_dict(),inplace=True)
#make sure there is no NaN any more
sum(np.isfinite(trainCleaned)==False) #return all zeros
sum(np.isfinite(testData)==False)
logging.warning('preprocessing done')

def marshall_palmer(ref, minutes_past):
    #print "Estimating rainfall from {0} observations".format(len(minutes_past))
    # how long is each observation valid?
    valid_time = np.zeros_like(minutes_past)
    valid_time[0] = minutes_past.iloc[0]
    for n in xrange(1, len(minutes_past)):
        valid_time[n] = minutes_past.iloc[n] - minutes_past.iloc[n-1]
    valid_time[-1] = valid_time[-1] + 60 - np.sum(valid_time)
    valid_time = valid_time / 60.0

    # sum up rainrate * validtime
    sum = 0
    for dbz, hours in zip(ref, valid_time):
        # See: https://en.wikipedia.org/wiki/DBZ_(meteorology)
        if np.isfinite(dbz):
            mmperhr = pow(pow(10, dbz/10)/200, 0.625)
            sum = sum + mmperhr * hours
    return sum

# each unique Id is an hour of data at some gauge
def testfunc(hour):
    #rowid = hour['Id'].iloc[0]
    # sort hour by minutes_past
    hour = hour.sort('minutes_past', ascending=True)
    est = marshall_palmer(hour['Ref'], hour['minutes_past'])
    return est

estimates = trainCleaned.groupby(trainCleaned.index).apply(testfunc)
x = trainCleaned.copy()
x['est'] = estimates
plt.yscale('log')
plt.xscale('log')
plt.scatter(x['Expected'],x['est'])
plt.xlabel('Expcted')
plt.ylabel('benchmarkest')
plt.savefig('alltrain_benchmarkest_expected_scatter.png')


# -*- coding: utf-8 -*-
"""
Created on Sun Dec  1 17:28:04 2019

@author: YUSS
"""

# -*- coding: utf-8 -*-
"""
Created on 27/5/2019
Interceptor V2
@author: I.
This ML challenge was created by SAMLA (National Electronic Warfare, Cyber & Intelligence Research Center) - a national research lab at Rafael http://portal.rafael.co.il/mlchallenge2019/Documents/samla.html

The goal of the game:
    Getting highest score in 100 games each running for 1000 steps.
    The player have access to 3 functions:

        Init(): This function initializes the game. It should be called before each game.

        Game_step(action_button): This function performs an action as described:
            action_button = 0: Change turret angle one step left
            action_button = 1: Do nothing in the current game step
            action_button = 2: Change turret angle one step right
            action_button = 3: Fire

            This function returns several variables:
                r_locs: Location of each rocket in the game (x,y)
                i_locs: Location of each interceptor in the game (x,y)
                c_locs: Location of each city in the game (x, width)
                ang: Turret angle
                score: Current player score

        Draw(): This function displays current game state (slows down your program. Not a must)

    Score is as follows:
        Rocket hits city: -15 points
        Rocket hits open field: -1 point
        Firing an interceptor: -1 point
        Intercepting a rocket: +4 points

In order to play, do the following:
***********************************

from Interceptor_V2 import Init, Draw, Game_step

Init()
for stp in range(1000):
    action_button = *** Insert your logic here: 0,1,2 or 3 ***
    r_locs, i_locs, c_locs, ang, score = Game_step(action_button)
    Draw()

*************************************

Don't forget to play by the rules described in the website.
"""

import numpy as np
import matplotlib.pyplot as plt
import cv2
import os
from mrcnn.config import Config
from mrcnn.model import MaskRCNN

class RocketConfig(Config):
    NAME = "rocket"
    IMAGES_PER_GPU = 1
    NUM_CLASSES = 2
#    STEPS_PER_EPOCH = 1000

config = RocketConfig()
config.IMAGE_SHAPE = [192, 384, 3]
config.IMAGE_MIN_DIM = 192
config.IMAGE_MAX_DIM = 384
config.IMAGE_RESIZE_MODE = 'none'
config.USE_MINI_MASK = False
config.display()

model = MaskRCNN(mode='inference', model_dir='./', config=config)
model.load_weights('./mask_rcnn_rocket_0070.h5', by_name=True)

class World():
    width = 10000 # [m]
    height = 4000 # [m]
    dt = 0.2 # [sec]
    time = 0 # [sec]
    score = 0
    reward_city = -15
    reward_open = -1
    reward_fire = -1
    reward_intercept = 4
    g = 9.8 # Gravity [m/sec**2]
    fric = 5e-7 # Air friction [Units of Science]
    rocket_prob = 1 # expected rockets per sec

class Turret():
    x = -2000 # [m]
    y = 0 # [m]
    x_hostile = 4800
    y_hostile = 0
    ang_vel = 30 # Turret angular speed [deg/sec]
    ang = 0 # Turret angle [deg]
    v0 = 800 # Initial speed [m/sec]
    prox_radius = 150 # detonation proximity radius [m]
    reload_time = 1.5 # [sec]
    last_shot_time = -3 # [sec]

    def update(self, action_button):
        if action_button == 0:
            self.ang = self.ang - self.ang_vel * world.dt
            if self.ang < -90: self.ang = -90

        if action_button == 1:
            pass

        if action_button == 2:
            self.ang = self.ang + self.ang_vel * world.dt
            if self.ang > 90: self.ang = 90

        if action_button == 3:
            if world.time - self.last_shot_time > self.reload_time:
                Interceptor()
                self.last_shot_time = world.time # [sec]

class Interceptor():
    def __init__(self):
        self.x = turret.x
        self.y = turret.y
        self.vx = turret.v0 * np.sin(np.deg2rad(turret.ang))
        self.vy = turret.v0 * np.cos(np.deg2rad(turret.ang))
        world.score = world.score + world.reward_fire
        interceptor_list.append(self)

    def update(self):
        self.v_loss = (self.vx ** 2 + self.vy ** 2) * world.fric * world.dt
        self.vx = self.vx * (1 - self.v_loss)
        self.vy = self.vy * (1 - self.v_loss) - world.g * world.dt
        self.x = self.x + self.vx * world.dt
        self.y = self.y + self.vy * world.dt
        if self.y < 0:
            Explosion(self.x, self.y)
            interceptor_list.remove(self)
        if np.abs(self.x) > world.width / 2:
            interceptor_list.remove(self)

class Rocket():
    def __init__(self, world):
        self.x = turret.x_hostile # [m]
        self.y = turret.y_hostile # [m]
        self.v0 = 700 + np.random.rand() * 300 # [m/sec]
        self.ang = -88 + np.random.rand() * 68 # [deg]
        self.vx = self.v0 * np.sin(np.deg2rad(self.ang))
        self.vy = self.v0 * np.cos(np.deg2rad(self.ang))
        rocket_list.append(self)

    def update(self):
        self.v_loss = (self.vx ** 2 + self.vy ** 2) * world.fric * world.dt
        self.vx = self.vx * (1 - self.v_loss)
        self.vy = self.vy * (1 - self.v_loss) - world.g * world.dt
        self.x = self.x + self.vx * world.dt
        self.y = self.y + self.vy * world.dt

class City():
    def __init__(self, x1, x2, width):
        self.x = np.random.randint(x1, x2) # [m]
        self.width = width # [m]
        city_list.append(self)
        self.img = np.zeros((200, 800))
        for b in range(60):
            h = np.random.randint(30, 180)
            w = np.random.randint(30, 80)
            x = np.random.randint(1, 700)
            self.img[0:h, x:x+w] = np.random.rand()
        self.img = np.flipud(self.img)

class Explosion():
    def __init__(self,x,y):
        self.x = x
        self.y = y
        self.size = 500
        self.duration = 0.4 # [sec]
        self.verts1 = (np.random.rand(30 ,2)- 0.5) * self.size
        self.verts2 = (np.random.rand(20 ,2)- 0.5) * self.size / 2
        self.verts1[:,0] = self.verts1[:,0] + x
        self.verts1[:,1] = self.verts1[:,1] + y
        self.verts2[:,0] = self.verts2[:,0] + x
        self.verts2[:,1] = self.verts2[:,1] + y
        self.hit_time = world.time
        explosion_list.append(self)

    def update(self):
        if world.time - self.hit_time > self.duration:
            explosion_list.remove(self)

def Check_interception():
    for intr in interceptor_list:
        for r in rocket_list:
            if ((r.x - intr.x)**2 + (r.y - intr.y)**2)**0.5 < turret.prox_radius:
                rocket_list.remove(r)
                Explosion(intr.x, intr.y)
                if intr in interceptor_list: interceptor_list.remove(intr)
                world.score = world.score + world.reward_intercept

def Check_ground_hit():
    for r in rocket_list:
        if r.y < 0:
            city_hit = False
            for c in city_list:
                if np.abs(r.x - c.x) < c.width:
                    city_hit = True
            if city_hit == True:
                world.score = world.score + world.reward_city
            else:
                world.score = world.score + world.reward_open
            Explosion(r.x, r.y)
            rocket_list.remove(r)

def Draw():
    plt.cla()
    plt.rcParams['axes.facecolor'] = 'black'
    for r in rocket_list:
        plt.plot(r.x, r.y,'.y')
    for intr in interceptor_list:
        plt.plot(intr.x, intr.y,'or')
        C1 = plt.Circle((intr.x, intr.y), radius=turret.prox_radius, linestyle='--', color='gray', fill=False)
        ax = plt.gca()
        ax.add_artist(C1)
    for c in city_list:
        plt.imshow(c.img, extent=[c.x-c.width/2, c.x+c.width/2, 0, c.img.shape[0]])
        plt.set_cmap('bone')
    for e in explosion_list:
        P1 = plt.Polygon(e.verts1, True, color='yellow')
        P2 = plt.Polygon(e.verts2, True, color='red')
        ax = plt.gca()
        ax.add_artist(P1)
        ax.add_artist(P2)
    plt.plot(turret.x, turret.y,'oc', markersize=12)
    plt.plot([turret.x, turret.x + 100*np.sin(np.deg2rad(turret.ang))],
             [turret.y, turret.y + 100*np.cos(np.deg2rad(turret.ang))],'c', linewidth = 3)
    plt.plot(turret.x_hostile, turret.y_hostile,'or', markersize=12)
    plt.axes().set_aspect('equal')
    plt.axis([-world.width / 2, world.width / 2, 0, world.height])
    #plt.title('Score: ' + str(world.score))

    plt.xticks([])
    plt.yticks([])

    plt.draw()

    plt.gcf().savefig('./im.jpg', bbox_inches = 'tight', pad_inches = 0)
    n_img = cv2.imread('./im.jpg')
    os.remove('./im.jpg')

    plt.pause(0.001)
    return n_img

def Init():
    global world, turret, rocket_list, interceptor_list, city_list, explosion_list
    world = World()
    rocket_list = []
    interceptor_list = []
    turret = Turret()
    city_list = []
    explosion_list = []
    City(-world.width * 0.5 + 400, -world.width * 0.25 - 400, 800)
    City(-world.width * 0.25 + 400, -400, 800)
    plt.rcParams['axes.facecolor'] = 'black'

def Game_step(action_button):

    world.time = world.time + world.dt
    rand_in = np.random.rand()
    if rand_in < world.rocket_prob * world.dt:
        Rocket(world)

    for r in rocket_list:
        r.update()

    for intr in interceptor_list:
        intr.update()

    for e in explosion_list:
        e.update()

    turret.update(action_button)
    Check_interception()
    Check_ground_hit()

    r_locs = np.zeros(shape=(len(rocket_list), 2))
    for ind in range(len(rocket_list)):
        r_locs[ind, :] = [rocket_list[ind].x, rocket_list[ind].y]

    i_locs = np.zeros(shape=(len(interceptor_list), 2))
    for ind in range(len(interceptor_list)):
        i_locs[ind, :] = [interceptor_list[ind].x, interceptor_list[ind].y]

    c_locs = np.zeros(shape=(len(city_list), 2))
    for ind in range(len(city_list)):
        c_locs[ind, :] = [city_list[ind].x, city_list[ind].width]

    return r_locs, i_locs, c_locs, turret.ang, world.score

def Create_state(n_img):

    #prepare image for prediction
    gray_im = cv2.cvtColor(n_img, cv2.COLOR_BGR2GRAY)
    gray_im = gray_im[:135, 4:]
    pad = np.zeros((192, 384)).astype('uint8')
    pad[29:-28, 25:-25] = gray_im
    color_im = cv2.cvtColor(pad, cv2.COLOR_GRAY2BGR)
    #state
    state=np.zeros((192,384))
    #predict
    y_h = model.detect([color_im])[0]
    #rois (bounding boxes)
    for y_min,x_min,y_max,x_max in y_h["rois"]:
    #take the center of each rocket (calculate middle of two points)
#        center_y,center_x=(y_min+y_max)/2,(x_min+x_max)/2
    #add one to rocket's center in state mat
        state[y_min:y_max+1, x_min:x_max+1] = 1
#        state[center_y,center_x]=1

    return state

#def start(iters):
#    Init()
#    np.random.seed(1)
#    table=list()
##    state = env.observation_space
#    for stp in range(iters):
#        action_button = np.random.randint(0,4)
##        action_button = RockeAgent.act(state)
##        --> agent(state) --> a
#        r_locs, i_locs, c_locs, ang, score = Game_step(action_button)
#        n_img=Draw()
#        state=Create_state(n_img)
#        table.append([state,action_button,score])

#start(15)

"""
TESTING CONVERSATION
im = cv2.imread('./train/85.jpg')
gray_im = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)

gray_im = gray_im[:135, 4:]
pad = np.zeros((192, 384)).astype('uint8')
pad[29:-28, 25:-25] = gray_im

color = cv2.cvtColor(pad, cv2.COLOR_GRAY2BGR)
plt.imshow(color)

im = plt.imread('./train/crop/85.jpg')
plt.imshow(im)
"""