#!/bin/env python
# -*- coding: utf-8 -*-
#    TOMUSS: The Online Multi User Simple Spreadsheet
#    Copyright (C) 2008-2014 Thierry EXCOFFIER, Universite Claude Bernard
#
#    This program is free software; you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation; either version 2 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
#    Contact: Thierry.EXCOFFIER@bat710.univ-lyon1.fr

import os
import time
import threading
import socket
import re
import collections
import cgi
from . import utilities
from . import configuration
from . import column
from . import data
from . import files
from . import inscrits
from .cell import CellValue, Lines, cellempty
from . import sender
from . import plugins

def check_table(t):
    cols = tuple(t.lines.lines.values())
    if cols:
        nr_cols = len(tuple(t.lines.lines.values())[0])
    else:
        return # can't check because there is no lines...
    warn('%s, %s, %s' % (t.ue, len(t.columns), nr_cols))
    if len(t.columns) != nr_cols:
        raise ValueError('bad table')

def date_time(date):
    if date is None:
        return '19700101010101'
    else:
        return time.strptime(date, '%Y%m%d%H%M%S')


warn = utilities.warn

tables = {}

the_head = files.add('FILES','head.html')

js = utilities.js

canceled_loads = []

# Key : student_id, give the tables ue name
# Used only in 'suivi' because it does not contain year/semester
tables_of_student = {}

class Page(object):
    def __init__(self, the_ticket, user_name, page_id, ttable,
                 user_ip, user_browser, date=None):
        self.ticket = the_ticket
        self.user_name = user_name
        self.page_id = page_id
        self.table = ttable
        self.request = 0
        self.browser_file = None
        self.user_ip = user_ip
        self.user_browser = user_browser
        self.date = date

    def check_identity(self, the_ticket, user_name, user_ip, user_browser):
        # Allow IP change with the same identity
        if (the_ticket == self.ticket
            and user_name == self.user_name
            and user_browser == self.user_browser):
            return
        warn('received ticket=%s user_name=%s user_browser=%s ip=%s' % (
            the_ticket, user_name, user_browser, user_ip), what="Info")
        warn('current  ticket=%s user_name=%s user_browser=%s ip=%s' % (
            self.ticket, self.user_name, self.user_browser, self.user_ip),
             what="Info")
        if configuration.regtest_sync or not configuration.regtest:
            raise ValueError('Cheater')

    def add_request(self, request, action, path, output_file):
        request_list.append((self.page_id, request, self, action, path,
                             output_file))

    def __str__(self):
        return '%s[%s]:%s(%s)' % (self.table, self.page_id, self.user_name,
                                  self.date)

    def date_time(self):
        return date_time(self.date)
    def day(self):
        return self.date[6:8] + '/' + self.date[4:6] + '/' + self.date[:4]

    def backtrace_html(self):
        return (cgi.escape(str(self)) + '<br>'
               + cgi.escape(self.user_browser) + '<br>'
                + self.user_ip)


def table_filename(year, semester, ue):
    return os.path.join(configuration.db, 'Y'+str(year), 'S'+semester, ue + '.py')

def filter_language(language):
    # Remove not translated languages and duplicates
    t = []
    for x in language.strip(",").split(','):
        if x in plugins.languages:
            if x not in t:
                t.append(x)
    return ','.join(t)

def get_preferences(user_name, create_pref=True, the_ticket=None):
    my_identity2 = utilities.login_to_module(user_name)
    prefs_table = table(0, 'Preferences', my_identity2, None, None,
                        create=create_pref)

    if prefs_table is None:
        p = {}
    else:
        p = prefs_table.template.preferences(prefs_table)
    warn('Language in preferences: (%s)' % p.get('language', ''), what="lang")
    if p.get('language', '') == '':
        if the_ticket is None:
            warn('Search old ticket to find language')
            for the_ticket in ticket.tickets.values():
                if the_ticket.user_name == user_name:
                    break
        warn('Language in ticket: (%s)' % the_ticket.language, what="lang")
        p['language'] = the_ticket.language

    p['language'] = filter_language(p['language'])
    warn('Language after filtering: ' + p['language'])
    if p['language'] == '':
        warn('Language in server: (%s)' % configuration.language,
             what="lang")
        p['language'] = configuration.language

    return p


#REDEFINE
# This function returns javascript code to be included
# in the header of the 'ue' table
def table_head_more(dummy_ue):
    return ''

class TT:
    def create(self, ttable):
        ttable.new_page('', data.ro_user, '', '')
        ttable.table_attr(ttable.pages[0],'masters',[ttable.user])

empty_template = TT()

def translations_init(language):
    languages = []
    language = filter_language(language + ',' + configuration.language)
    for lang in language.split(','):
        languages.append(
            '<script onload="this.onloadDone=true;" src="%s/%s.js"></script>'
            % (configuration.url_files, lang))
    return ('<script>var translations = {},'
            + 'preferences={"language":%s} ; </script>\n'
            % js(language)
            + '\n'.join(languages) + '\n')

def table_head(year=None, semester=None, the_ticket=None,
               user_name='', page_id=-1, ue='',
               create_pref=True,
               attrs_from=0, hide_more=False, table=None):
    s = configuration.suivi.url(year, semester, the_ticket)
    t = ticket.tickets.get(the_ticket, None)
    prefs_table = get_preferences(user_name, create_pref, the_ticket=t)
    try:
        background = configuration.semesters_color[configuration.semesters.index(semester)]
        background = '<style>BODY, TABLE INPUT, #current_input, BODY TABLE.colored TD { background-color: ' + background + '}</style>'
    except ValueError:
        background = ''

    my_identity2 = utilities.login_to_module(user_name)

    return (str(the_head) + background +
            translations_init(prefs_table['language']) +
            '<script>\n' +
            'page_id = "%d" ;\n' % page_id +
            'my_identity = %s ;\n' % repr(user_name) +
            'my_identity2 = %s ;\n' % repr(my_identity2) +
            'url = %s ;\n' % js(configuration.server_url) +
            'year = "%s" ;\n' % year +
            'semester = "%s" ;\n' % semester +
            'ticket = "%s" ;\n' % the_ticket +
            'ue = "%s" ;\n' % ue +
            'suivi = %s ;\n' % js(s) +
            'version = "%s" ;\n' % configuration.version +
            'root = %s ;\n' % js(list(configuration.root)) +
            'cas_url = %s ;\n' % repr(configuration.cas) +
            'preferences = %s ;\n' % prefs_table +
            'lines = {};\n' +
            'columns = [];\n' +
            'lines_to_load = 0 ;\n' +
            'ticket_time_to_live = %d ;\n' % configuration.ticket_time_to_live +
            'max_visibility_date = %d ;\n' % configuration.max_visibility_date +
            'gui_record = %d ;\n' % int(configuration.gui_record) +
            'check_down_connections_interval = %d ;\n' % configuration.check_down_connections_interval +
            'table_attr = {\n' +
                ',\n'.join(attr.name+':'+js(
                getattr(attrs_from, attr.name,
                        attr.get_default_value(table)))
                for attr in column.TableAttr.attrs.values()
                ) + '} ;\n' +
            (hide_more and '' or table_head_more(ue)) +
            '</script>\n')

def import_template(names):
    for name in names:
        if configuration.regtest and name[0] == 'LOCAL':
            continue
        filename = os.path.join(*name) + '.py'
        if os.path.exists(filename):
            return utilities.import_reload(filename)[0]
    return None

class Table(object):
    new_abjs = None
    force_update = 0
    
    def __init__(self, year, semester, ue, ro=True, user=None):

        warn('%s/%s/%s ro=%s user=%s' % (year, semester, ue, ro, user),
             what="table")
        self.year = year
        self.semester = semester
        self.ue = ue
        self.loading = False # The table is not being loaded
        self.sent_to_browsers = [] # Nothing has been sent

        if self.ue == 'abj' or self.ue == 'data':
            # Break the import in the file
            raise ValueError("Invalid table name: '%s'" % self.ue)

        # Remove -1 -2 -3 at the end of the UE name
        if len(ue) > 4 and ue[-1].isdigit() and ue[-2] == '-':
            self.ue_code = ue[:-2]
        elif (len(ue) > 5
              and ue[-1].isdigit() and ue[-2].isdigit() and ue[-3] == '-'):
            self.ue_code = ue[:-3]
        else:
            self.ue_code = ue

        for attr in column.TableAttr.attrs.values():
            d = attr.get_default_value(self)
            if isinstance(d, list):
                d = list(d)
            elif isinstance(d, dict):
                d = dict(d)
            setattr(self, attr.name, d)

        self.pages = []
        self.active_pages = []
        self.columns = column.Columns(self)
        self.lines = Lines(self.columns)
        self.the_lock = threading.Lock()
        self.ro = ro
        self.mtime = 0
        self.the_key_dict = collections.defaultdict(list)
        self.unloaded = False
        self.do_not_unload = []
        dirname = os.path.join(configuration.db,
                               'Y'+str(self.year), 'S'+self.semester)
        self.filename = os.path.join(dirname, ue + '.py')
        self.is_extended = os.path.islink(self.filename)

        self.template = import_template((
            ('LOCAL', 'LOCAL_TEMPLATES', self.ue_code),
            ('TEMPLATES', self.ue_code),
            ('LOCAL', 'LOCAL_TEMPLATES', self.semester),
            ('TEMPLATES', self.semester),
            ))
        if self.template is None:            
            self.template = empty_template

        if hasattr(self.template, 'init'):
            self.template.init(self)
            
        for v in self.template.__dict__.values():
            if isinstance(v, utilities.Variables):
                self.group = v._group

        created = False
        self.on_disc = True
        if self.modifiable:
            utilities.mkpath_safe(dirname)
            if not os.path.exists(self.filename):
                s = ['# -*- coding: utf8 -*- ']
                s.append('from TOMUSS.data import *')
                s = '\n'.join(s) + '\n'
                utilities.mkpath_safe(dirname)
                utilities.append_file_safe(self.filename, s)
                created = True
        else:
            if not os.path.exists(self.filename):
                self.on_disc = False

        # Remove final .py
        self.module = 'TOMUSS.' + self.filename[:-3].replace(os.path.sep,'.')

        if not os.path.exists(self.filename):
            # Read only table and file does not exists
            return

        self.mtime = os.path.getmtime(self.filename)

        if not created:
            data.begin(self)
            try:
                __import__(self.module)
                utilities.unload_module(self.module)
            finally:
                data.end()

        else:
            if ro:
                return # Do not create the table
            # Remove this file because it is created the same second
            # than the .py file, So it is not recompiled in some case.
            # os.remove(self.filename + 'c')
            warn('Create start', what='table')
            self.user = user # Usable by template.create()
            # We lock because table change methods expects to be locked.
            # But this lock does not protect anything
            self.lock()
            try:
                if hasattr(self.template, "create"):
                    self.template.create(self)
                else:
                    empty_template.create(self)
            finally:
                self.unlock()
                warn('Unlock', what='table')
            warn('Create end', what='table')
            self.pages[0].request = 1 # Not an empty page, it's for the stats

        if self.template and hasattr(self.template, 'onload'):
            self.template.onload(self)
            
        if not ro:
            # Check mails or new students
            # Only mail if allow_modification is False
            warn('Update student list', what='table')
            update_students.append(self)

        if ro and self.official_ue and not configuration.index_are_computed:
            # Student are never removed, but it is not important
            for login in self.the_keys():
                if login in tables_of_student:
                    if self.ue not in tables_of_student[login]:
                        tables_of_student[login].add(self.ue)
                else:
                    tables_of_student[login] = set((self.ue,))

        if self.is_extended:
            self.destination_is_modifiable = self.modifiable
            # To forbid the edit of the same table with 2 names
            self.modifiable = 0

    def update(self):
        """Update the table if the file on disc changed.
        It is used only by 'suivi' servers
        """
        # warn('Start ro=%s dt=%f p=%s' % (self.ro, time.time() - self.mtime, os.path.exists(self.filename)), what="table")
        if not self.ro:
            return self

        if not os.path.exists(self.filename):
            # Do not work (to to recreate it)
            # tables_manage("del", self.year, self.semester, self.ue)
            # Should return None, but as it is very intrusive
            # Make a non dangerous kludge
            self.official_ue = False
            return self
        
        if time.time() - self.mtime < configuration.maximum_out_of_date:
            return self

        if os.path.getmtime(self.filename) <= self.mtime:
            return self

        warn('%s time=%s mtime=%s' % (self.ue, time.time(), self.mtime),
             what="table")

        return tables_manage("replace", self.year, self.semester, self.ue,
                             new_table = Table(self.year, self.semester,
                                               self.ue, ro=True))


    def change_mails(self, mails):
        self.table_attr_computed('mails', mails)
    def change_portails(self, portails):
        self.table_attr_computed('portails', portails)

    def table_attr_computed(self, attr, value):
        if getattr(self, attr) == value:
            return
        setattr(self, attr, value)
        t = '<script>Xtable_attr(%s,%s);</script>\n' % (repr(attr), js(value))
        self.send_update(None, t)

    def update_mail(self, login, mail):
        warn('update mail of ' + self.location(), what="table")
        self.mails[login] = mail
        self.send_update(None,
                         ('<script>update_mail(' + repr(login)
                         + ',' + js(mail) + ');</script>\n').encode('utf-8')
                         )

    def update_portail(self, login, portail): 
        self.portails[login] = portail
        self.send_update(None,
                         ('<script>update_portail(' + repr(login)
                         + ',' + js(portail) + ');</script>\n').encode('utf-8')
                         )
        
    def lock(self):
        warn('start', what='debug')
        sender.send_live_status('<script>b("%d/%s/%s");</script>\n' %
                                (self.year, self.semester, self.ue))
        self.the_lock.acquire()
        warn('acquired', what='debug')

    def unlock(self):
        warn('start', what='debug')
        self.the_lock.release()
        sender.send_live_status('<script>r("%d/%s/%s");</script>\n' %
                                (self.year, self.semester, self.ue))

    def panic(self, txt):
        utilities.send_backtrace('UE: %s ' % self.ue + txt,
                                 exception=False)
        self.send_update(None, "<script>window.close();</script>\n")
        raise ValueError('PANIC')

    def state_is_fine(self):
        if self.unloaded:
            self.panic('Modification on an unloaded table')
        if not self.the_lock.locked():
            self.panic('Modification on an unlocked table')
        if self.is_extended:
            self.panic('Modification on an extended table')
        if configuration.read_only:
            self.panic('Modification in a readonly process')
                
    def log(self, text):
        self.state_is_fine()
        utilities.append_file_safe(self.filename, text + '\n')
        warn(self.filename + ' ' + text)
        self.mtime = time.time()

    def new_page(self, ticket, user_name, user_ip, user_browser, date=None):
        if not self.loading and self.modifiable:
            self.log('new_page(%s ,%s, %s, %s, %s) # %d' % (
                repr(ticket),
                repr(user_name),
                repr(user_ip),
                repr(user_browser),
                repr(date),
                len(self.pages),
                ))
        p = Page(ticket, user_name, len(self.pages), self,
                 user_ip, user_browser, date)
        p.logged = self.loading or self.modifiable
        p.index = len(self.sent_to_browsers) - 1
        self.pages.append(p)
        return p

    def active_page(self, page, ffile):
        warn('page=%s file=%s(%s)' % (page, ffile, ffile.closed), what="table")
        page.browser_file = ffile
        if page not in self.active_pages:
            self.active_pages.append(page)

    def remove_active_page(self, page):
        self.active_pages.remove(page)
        page.index = getattr(page.browser_file, 'index', page.index)
        page.browser_file = None # stop a memory leak
            
    def send_update(self, page, value, store=True):
        warn('actives: %s' % str(self.active_pages), what='table')
        if store:
            self.sent_to_browsers.append(value)
        for p in tuple(self.active_pages):
            if p == page:
                continue
            if p.browser_file.closed:
                self.remove_active_page(p)
                if not hasattr(p, 'end_of_load'):
                    global canceled_loads
                    # Update list of canceled page load
                    now = time.time()
                    canceled_loads.append((now, p.user_name))
                    old = now - 3600
                    canceled_loads = [i for i in canceled_loads
                                     if i[0] > old]
                    user = p.user_name
                    nr = list([i for i in canceled_loads if i[1] == user])
                    if len(nr) == 4: # 4 cancels in less than one hour
                        user_mail = inscrits.L_fast.mail(user)
                        if user_mail:
                            utilities.send_mail_in_background(
                                user_mail,
                                # XXX need translation
                                utilities._("MSG_document_problems"),
                                unicode(utilities.read_file(
                                        os.path.join('FILES', 'mail_cancel')),
                                        'utf8').encode('latin1')
                                )

                    # Send a mail to the maintainer
                    utilities.send_backtrace(
                        repr(nr),
                        utilities._("MSG_document_canceled") + user,
                        exception=False)
                        
                continue
            if value is True:
                sender.append(p.browser_file, self.content(p),
                              index=len(self.sent_to_browsers))
            else:
                sender.append(p.browser_file, value,
                              index=len(self.sent_to_browsers))

    def authorized(self, user_name, value):
        # Authorized because the test have yet be done in the past
        if self.loading:
            return True
        # Values setted by user '' are modifiable
        if value.author == data.rw_user:
            return True
        # Empty values are modifiable by anyone
        if value.value == '':
            return True
        # The user can change its values
        if value.author == user_name:
            return True
        # RO and RW users have all the rights
        if user_name == data.ro_user or user_name == data.rw_user:
            return True
        # Values setted by user '*' are not modifiable
        if value.author == data.ro_user:
            return False
        # The masters of the UE may change any value setted by another user
        if user_name in self.masters:
            return True
        return False

    def authorized_column(self, user_name, a_column):
        return self.authorized(
            user_name, CellValue(a_column.title, a_column.author))

    def cell_change(self, page, col, lin, value=None,
                    date=None, force_update=False,
                    change_author=True):

        if not self.loading and not self.modifiable:
            return self.bad_ro(page)

        a_column = self.columns.from_id(col)
        if a_column == None:
            raise ValueError(utilities._("MSG_document_bug_column") % col)

        on_a_new_line = lin not in self.lines
        line = self.lines[lin]
        cell = line[a_column.data_col]

        if value is None:
            value = cell.value

        if not self.loading and not force_update:
            if not self.authorized(page.user_name, cell):
                utilities.warn('cell value = (%s)' % cell.value)
                return self.bad_auth(page, "cell_change %s/%s/%s" % (
                        col, lin, value))
            if a_column.locked and page.user_name != data.ro_user:
                self.error(page, utilities._("MSG_document_column_locked"))
                return 'bad.png'

        old_value = str(cell.value)
        new_value = str(value)
        # If the old value is the float 10
        # And the new one the string 10
        # The string compare will be false because "10.0" != "10"
        # So we also compare non string values.
        try:
            equal = float(cell.value) == float(value)
        except ValueError:
            equal = old_value == new_value
        if equal:
            if cell.author == page.user_name:
                return 'ok.png'
            if not change_author:
                return 'ok.png'

        # if isinstance(value, str) and value.find('.') != -1:
        if a_column.type.name == 'Note':
            try:
                value = float(value)
            except ValueError:
                pass

        if (not self.loading
            and a_column.repetition
            and value != ''
            and not equal
            ):
            n = 0
            data_col = a_column.data_col
            if a_column.repetition > 0:
                verify_lines = self.lines.values()
            else:
                grp = line[self.columns.get_grp()].value
                seq = line[self.columns.get_seq()].value
                verify_lines = list(self.columns.table.lines_of_grp(grp, seq))
            for a_line in verify_lines:
                if a_line[data_col].value == value:
                    n += 1
            if n >= abs(a_column.repetition):
                sender.append(
                    page.browser_file,
                    '<script>alert(_("ALERT_repetition")+%s+"/"+%s);</script>\n'
                    % (js(value), js(a_column.title)))
                return 'bad.png'


        if (not self.loading and self.template
            and not equal
            and hasattr(self.template, 'cell_change')):
            self.template.cell_change(self, page, col, lin, value, date)

        if a_column.data_col == 0:
            login = utilities.the_login(old_value)
            if login in self.the_key_dict:
                try:
                    self.the_key_dict[login].remove(lin)
                    if len(self.the_key_dict[login]) == 0:
                        del self.the_key_dict[login]
                except:
                    if login:
                        utilities.warn(str(page) + ' old_login=' + login
                                       + ' new_login=' + new_value
                                       + ' lin=' + lin
                                       + ' ' + repr(self.the_key_dict[login]))
            new_login = utilities.the_login(new_value)
            self.the_key_dict[new_login].append(lin)
            if not self.loading and self.official_ue:
                indexes_to_update.append((self, login, new_login))
        elif on_a_new_line:
            self.the_key_dict[''].append(lin)
            

        # The class may change on value change
        cell = line[a_column.data_col] = cell.set_value(value=value,
                                                        author=page.user_name,
                                                        date=date)
        if not self.loading:
            self.log('cell_change(%s,%s,%s,%s,"%s")' % (
                page.page_id,
                repr(col),
                repr(lin),
                repr(value),
                cell.date))
            t = '<script>Xcell_change(%s,%s,%s,%s,%s,%s);</script>\n' % (
                js(col),
                js(lin),
                js(value),
                js(cell.date),
                js(cell.author),
                js(cell.history),
                )
            if force_update:
                self.send_update(None, t)
            else:
                self.send_update(page, t)
            cell_changed_list_fast.append((self, lin, a_column))
        else:
            page.request += 1

        return 'ok.png'

    def default_nr_columns_change(self, n):
        """Deprecated"""
        self.default_nr_columns = n

    def add_master(self, name, page_id=0):
        """Deprecated"""
        name = name.lower()
        if name in self.masters:
            self.masters.remove(name)
        else:
            self.masters.append(name)
        if not self.loading:
            # For old TEMPLATES files
            self.log('table_attr("masters",%d,%s)' % (page_id, repr(name))) 

    def get_a_master_page(self):
        """Returns a page modifiable by the masters.
        Create one if there is none"""
        rw = None
        for p in self.pages:
            if p.user_name in self.masters:
                return p
            if p.user_name == data.rw_user:
                rw = p
        if self.masters:
            return self.new_page('', self.masters[0], '', '')
        else:
            if rw:
                return rw
            return self.new_page('', data.rw_user, '', '')

    def private_toggle(self, page):
        """Deprecated"""
        self.private = 1 - self.private
        if not self.loading: # compatibility with old TEMPLATES
            self.log('table_attr("private",%d,%d)' % (page.page_id,
                                                      self.private))
        return 'ok.png'

    def error(self, page, message, more_in_mail=""):
        utilities.send_backtrace(
            'UE: %s, Page: %s' % (self.ue, page)
            + '\n' + more_in_mail, subject='###' + message,
            exception=False)
        if '_(' not in message:
            # The message is not javascript program
            message = js(message)
        message = message + '+"\\n\\n"+_("ERROR_server_bug")'
        sender.append(page.browser_file,
                      '<script>alert(%s);</script>\n' % message)
        return "bad.png"

    def bad_column(self, page):
        return self.error(page, '_("ALERT_column_not_exist")')

    def bad_auth(self, page, more_in_mail=""):
        return self.error(page, '_("ALERT_not_authorized")', more_in_mail)

    def bad_ro(self, page):
        return self.error(page, '_("ALERT_value_ro")')

    def comment_change(self, page, col, lin, value):
        if not self.loading and not self.modifiable:
            return self.bad_ro(page)

        a_column = self.columns.from_id(col)
        if a_column == None:
            return self.bad_column(page)

        line = self.lines[lin]
        if value == line[a_column.data_col].comment:
            return 'ok.png'

        
        if not self.loading:
            if not self.authorized(page.user_name,
                                   line[a_column.data_col]):
                return self.bad_auth(page, "comment_change %s/%s/%s" % (
                        col, lin, value))
            if hasattr(self.template, 'comment_change'):
                self.template.comment_change(self, page, col, lin, value)
            self.log('comment_change(%s,%s,%s,%s)' % (
                page.page_id,
                repr(col),
                repr(lin),
                repr(value),
                ))
            t = '<script>Xcomment_change(%s,%s,%s,%s);</script>\n' % (
                js(page.user_name),
                js(col),
                js(lin),
                js(value),
                )
            self.send_update(page, t)
        else:
            page.request += 1

        line[a_column.data_col] = line[a_column.data_col].set_comment(value)

        return 'ok.png'

    def empty(self, empty_even_if_used_page=False,
              empty_even_if_created_today=False,
              empty_even_if_column_created=False,
              ):
        if not empty_even_if_used_page and self.active_pages:
            return False, utilities._("MSG_document_table_active")
        if not empty_even_if_created_today and time.time()-self.mtime <24*3600:
            return False, utilities._("MSG_document_table_today")
        if self.comment:
            return False, utilities._("MSG_document_table_comment"
                                      ) % cgi.escape(self.comment)
        if not empty_even_if_column_created:
            for c in self.columns:
                if c.author != data.ro_user:
                    return False, utilities._("MSG_document_table_title")
        for line in self.lines.values():
            for j in line:
                if not j.empty():
                    return False, utilities._("MSG_document_table_cell")
                if j.comment:
                    return False, utilities._("MSG_document_table_cell_comment")

        return True, utilities._("MSG_document_table_lines") % len(self.lines)

    def problem_in_column_name(self):
        names = set()
        for c in self.columns:
            if c.title in names:
                return utilities._("MSG_document_table_duplicate") + c.title
            names.add(c.title)
        for c in self.columns:
            for name in c.depends_on():
                if name not in names:
                    return utilities._("MSG_document_table_no_column") % (
                        c.title, name)
        return ""
        

    def table_comment(self, page, comment):
        """Deprecated"""
        return self.table_attr(page, 'comment', comment)

    def add_empty_column(self, page, the_id):
        if self.columns.columns:
            position = max([c.position for c in self.columns]) + 1
        else:
            position = 0

        a_column = column.Column(the_id, page.user_name, position=position)

        self.columns.append(a_column)
        for line in self.lines.values():
            line.append(cellempty)
        return a_column

    def column_attr(self, page, col_id, attr, value):
        col = self.columns.from_id(col_id)
        if col is None:
            col = self.add_empty_column(page, col_id)
        return column.ColumnAttr.attrs[attr].set(self, page, col, value)

    def column_changed(self, a_column, attribute):
        "An column attribute change, column content may be updated"
        column_changed_list.append((self, a_column, attribute))

    def table_attr(self, page, attr, value):
        return column.TableAttr.attrs[attr].set(self, page, value)

    def column_comment(self, page, col_id, comment):
        """DEPRECATED : use column_attr"""
        self.column_attr(page, col_id, 'comment', comment)

    def column_change(self, page, col_id, title, ttype, test, weight,
                      freezed, hidden=0, width=1):
        """DEPRECATED : use column_attr"""

        col = self.columns.from_id(col_id)
        if col is None:
            col = self.add_empty_column(page, col_id)
        attrs = column.ColumnAttr.attrs
        attrs['type'].set(self, page, col, ttype)
        attrs['title'].set(self, page, col, title)
        attrs['freezed'].set(self, page, col, freezed)
        attrs['hidden'].set(self, page, col, hidden)
        attrs['width'].set(self, page, col, width)

        t = test.split(';')
        if len(t) == 4:
            test = '[' + t[0].strip('[') + ';' + t[3].strip(']') + ']'
            attrs['red'].set(self, page, col, t[1])
            attrs['green'].set(self, page, col, t[2])
        if ttype == 'Nmbr':
            attrs['test_filter'].set(self, page, col, test)
        else:
            if test:
                attrs['minmax'].set(self, page, col, test)

        t = re.split(' +', weight)
        if len(t) > 1:
            weight = t[0]
            attrs['columns'].set(self, page, col, ' '.join(t[1:]))

        if weight:
            attrs['weight'].set(self, page, col, weight)


    def column_delete(self, page, col):
        if not self.loading and not self.modifiable:
            return self.bad_ro(page)
        a_column = self.columns.from_id(col)
        if a_column == None:
            return self.bad_column(page)
        # The first 'if' is to allow teachers to destroy
        # Columns created by 'data.ro_user' if they are not firsts ones.
        # THIS SHOULD BE REMOVED, but not yet.
        # It was tested because bad columns were created.
        if page.user_name not in self.masters and a_column.data_col < 6 :
            if not self.authorized_column(page.user_name, a_column):
                return self.bad_auth(page, "column_delete %s" % col)
        if (a_column.type.cell_is_modifiable
            and not a_column.empty_of_user_values()):
            return self.error(page, '_("ALERT_delete_not_empty")')

        if not self.loading:
            self.log('column_delete(%s,%s)' % (page.page_id, repr(col)))
            t = '<script>Xcolumn_delete(%s,%s);</script>\n' % (
                js(page.user_name), js(col))
            self.send_update(page, t)
        else:
            page.request += 1

        for line in self.lines.values():
            line.pop(a_column.data_col)
        self.columns.pop(a_column.data_col)
            
        return 'ok.png'

    def column_inscrit(self):
        for i, c in enumerate(self.columns):
            if c.freezed == 'C':
                return i
        return None

    def content_head(self, page):
        return table_head(
            self.year, self.semester, page.ticket, page.user_name,
            page.page_id, self.ue,
            attrs_from=self, table=self
            )

    def date_change(self, page, date):
        """Deprecated"""
        return column.TableAttr.attrs['dates'].set(self, page, date)

    def logins(self):
        for v in self.lines.values():
            if v[0].value != '':
                yield v[0].value

    def logins_valid(self):
        
        if self.column_inscrit() is None:
            for v in self.lines.values():
                if v[0].value != '':
                    yield v[0].value
        else:
            for v in self.lines.values():
                if v[0].value != '' and v[5].value != 'non':
                    yield v[0].value

    def the_keys(self):
        """Returns a dictionary for fast access by student login"""
        return self.the_key_dict

    def get_lines(self, login):
        login = utilities.the_login(login)
        if login in self.the_key_dict:
            for lin in self.the_key_dict[login]:
                yield self.lines[lin]
        return

    def lines_of_grp(self, grp, seq):
        if grp == '':
            return # Do not auto compute ABINJ??? if there is no real groups
        grp_col = self.columns.get_grp()
        seq_col = self.columns.get_seq()
        if grp_col and seq_col:
            for line in self.lines.values():
                if line[grp_col].value == grp and line[seq_col].value == seq:
                    yield line

    def get_items(self, login):
        login = utilities.the_login(login)
        if login in self.the_key_dict:
            for lin in self.the_key_dict[login]:
                yield lin, self.lines[lin]
        return
    
    def authors(self):
        """List of all the logins that have modified a cell value"""
        a = {}
        for v in self.lines.values():
            for cell in v:
                if cell.author not in ('', data.ro_user):
                    a[cell.author] = True
        return list(a.keys())

    def update_columns(self, columns, ro_page=None):
        """Update the default columns of the table.
        This can be called by the TEMPLATE 'check' method
        """
        if not self.modifiable:
            return
        if ro_page is None:
            ro_page = self.pages[0]
        locked = self.the_lock.locked()
        try:
            if not locked:
                self.lock()
            for col in sorted(columns):
                for attr, value in columns[col].items():
                    value = str(value)
                    a_column = self.columns.from_id(col)
                    if a_column is None or value!= str(getattr(a_column, attr)):
                        self.column_attr(ro_page, col, attr, value)
        finally:
            if not locked:
                self.unlock()

    def content(self, page):
        warn('%s content for page %d' % (
            self.filename, page.page_id))
        s = []

        s.append(self.content_head(page))

        try:
            s.append('  <script><!--')
            s.append(utilities.wait_scripts())
            s.append('--></script> ')
            lines = self.lines.js()
            i = 0
            nb = 100
            to_load = len(lines)
            while lines:
                s.append('<script><!--')
                s.append('\n'.join(lines[:nb]))
                del lines[:nb]
                i += nb
                s.append('--></script><div class="loading_bar"><div style="width:%d%%">&nbsp;</div></div>'
                         % int(i*100./to_load) )
            s.append('  <style>DIV.loading_bar { display: none;}</style>')
            s.append('''  <script><!--
            function initialize()
            {
            if ( ! wait_scripts("initialize()") )
               return ;
            var t = document.getElementsByTagName("DIV");
            for(var i=0; i<t.length; i++)
                if ( t[i].className == "loading_bar" )
                    t[i].parentNode.removeChild(t[i]) ;
            document.write(head_html()) ;
            insert_middle();
            ''')
            s.append(self.columns.js(hide=False))

            s.append('document.write(tail_html());')
            s.append('runlog(columns, lines) ;')
            s.append('set_updating(%s) ;' % int(self in update_students))
            s.append('}')
            if self.new_abjs:
                s.append(self.new_abjs)
            if self.template and hasattr(self.template, 'content'):
                s.append(self.template.content(self))
            s.append('initialize();')
            s.append('--></script>  \n')
            if self.template and hasattr(self.template, 'css'):
                s.append('<style>' + self.template.css + '</style>')
        finally:
            pass

        return '\n'.join(s)

    def location(self):
        return '%d %s %s' % (self.year, self.semester, self.ue)

    def rewrite(self, only_columns=False, user_name=''):
        s = ['# -*- coding: utf8 -*-']
        s.append('from data import *')
        s.append("new_page('' ,'" + data.ro_user + "', '', '')")
        pages = collections.defaultdict(lambda: 1)
        pages[data.ro_user] = 0
        if user_name:
            s.append("new_page('' ,'%s', '', '')" % user_name)
        else:
            authors = set()
            for c in self.columns:
                authors.add(c.author)
            for line in self.lines.values():
                for cell in line:
                    authors.add(cell.author)
            authors.discard(data.ro_user)
            for author in authors:
                pages[author] = len(pages)
                s.append("new_page('' ,'%s', '', '')" % author)
            
        for c in self.columns:
            # if c.empty() and c.type.cell_compute == 'undefined':
            #    continue
            for attr in column.ColumnAttr.attrs.values():
                if attr.name == 'author':
                    continue
                attr_value = attr.decode(getattr(c, attr.name))
                if attr_value != attr.default_value:
                    s.append('column_attr(%s,%d,%s,%s)' % (
                        repr(attr.name), pages[c.author],
                        repr(c.the_id), repr(attr_value)))
                         
        for attr in column.TableAttr.attrs.values():
            if attr.computed:
                continue
            if attr.name in ('modifiable', 'dates'):
                continue
            attr_value = getattr(self, attr.name)
            if attr.name == 'masters':
                if user_name not in attr_value:
                    attr_value.append(user_name)
                if '' in attr_value:
                    attr_value.remove('')
            attr_value = attr.decode(attr_value)
            if attr_value != attr.default_value:
                s.append('table_attr(%s,0,%s)' % (
                    repr(attr.name),
                    repr(attr_value)))

        if only_columns:
            return '\n'.join(s) + '\n'
                         
        for line_key, line in self.lines.items():
            for col, cell in zip(self.columns, line):
                if cell.value != '':
                    s.append('cell_change(%d,%s,%s,%s,%s)' % (
                            pages[cell.author],
                            repr(col.the_id),
                            repr(line_key),
                            repr(cell.value),
                            repr(cell.date),
                            ))
                    if cell.comment:
                        s.append('comment_change(%d,%s,%s,%s)' % (
                                pages[cell.author],
                                repr(col.the_id),
                                repr(line_key),
                                repr(cell.comment),
                                ))
                                                    
        return '\n'.join(s) + '\n'

    def close_active_pages(self):
        for page in self.active_pages:
            try:
                page.browser_file.write('<script>window.parent.close();</script>')
                page.browser_file.close()
            except:
                pass
        self.active_pages = []

    def do_not_unload_add(self, value):
        if value:
            self.do_not_unload.append(value)
    def do_not_unload_remove(self, value):
        self.do_not_unload.remove(value)

    def unload(self, force=False):
        if force:
            self.close_active_pages()
            self.do_not_unload = []
        if '*' in ''.join(self.do_not_unload):
            return
        warn(str(self.ue), what="table")

        # XXX
        # In rare cases the table is unloaded while the student list
        # is being updated. So we finish with a table half updated
        # But it is not important.
        deleted = tables_manage("del", self.year, self.semester, self.ue)
        try:
            if deleted:
                update_students.remove(self) # No update student list.
        except ValueError:
            pass
        return deleted

    def delete(self):
        warn(str(self.ue), what="table")
        self.unload(force=True)
        if not self.unloaded:
            return
        # XXX Not locked, so the table may be reloaded before deletion....
        utilities.unlink_safe(self.filename)
        utilities.unlink_safe(self.filename + 'c', do_backup=False)

        for name in self.masters:
            self.master_of_update('-', name)
        for login in self.the_keys():
            indexes_to_update.append((self, login, ''))


    def send_alert(self, text):
        self.send_update(None, '<script>alert(_("ALERT_message_for")+%s);</script>\n' %
                         js(self.ue + '\n\n' + text))

    def __str__(self):
        return '%d/%s/%s' % (self.year, self.semester, self.ue)

    def backtrace_html(self):
        return "Table: " + str(self)

    def readable_by(self, the_ticket):
        if (self.private
            and the_ticket.user_name not in self.masters 
            and the_ticket.user_name not in configuration.root
            ):
            warn('Unauthorized access', what='table')
            return False
        return True

    def master_of_update(self, what, name):
        if self.semester in configuration.master_of_exceptions:
            return
        d = utilities.manage_key('LOGINS', os.path.join(name, 'master_of'))
        if d is False:
            d = []
        else:
            d = eval(d)

        t = (str(self.year), self.semester, self.ue)
        if (t in d) and what == '-':
            d.remove(t)
        elif (t not in d) and what == '+':
            d.append(t)
        else:
            return

        utilities.manage_key('LOGINS', os.path.join(name, 'master_of'),
                             content = repr(d))

    def the_abjs(self):
        from . import abj

        grp_col = self.columns.get_grp()
        seq_col = self.columns.get_seq()
        t = []
        for login in self.logins():
            tt = abj.tierstemps(login)
            student = abj.Abj(self.year, self.semester, login)
            line = list(self.get_lines(login))
            if line:
                line = line[0]
                the_abjs= abj.do_prune(
                    student.abjs,
                    self.dates[0], self.dates[1]+86400,
                    grp_col and line[grp_col].value or '',
                    seq_col and line[seq_col].value or '',
                    self.ue)
            else:
                the_abjs = ()
            da = student.da
            if tt or the_abjs or da:
                t.append("%s:[[%s],[%s],%s]" % (
                    js(login),
                    ','.join(['[%s,%s,%s]' % (js(a),js(b),js(d))
                              for a,b,dummy_c,d in the_abjs]),
                    ','.join(['[%s,%s,%s]' % (js(a),js(b),js(d))
                              for a,b,dummy_c,d in da]),
                    js(tt.encode('utf-8'))))

        self.new_abjs = 'change_abjs({%s});\n' % ',\n'.join(t)
        return self.new_abjs

    def update_the_abjs(self):
        old_abjs = self.new_abjs
        if self.the_abjs() != old_abjs:
            self.send_update(None,'<script>' + self.new_abjs + '</script>')

    def link_to(self):
        ln = os.readlink(self.filename)[:-3].split(os.path.sep)
        ue = ln[-1]
        if len(ln) == 3:
            assert(ln[0] == '..')
            assert(ln[1][0] == 'S')
            year = self.year
            semestre = ln[1][1:]
        elif len(ln) == 5:
            assert(ln[0] == '..')
            assert(ln[1] == '..')
            assert(ln[2][0] == 'Y')
            assert(ln[3][0] == 'S')
            year = int(ln[2][1:])
            semestre = ln[3][1:]
        else:
            assert(len(ln) == 1)
        return year, semestre, ue

def send_alert(text):
    for atable in tables_values():
        atable.send_alert(text)

@utilities.add_a_lock
def tables_manage(action, year, semester, ue, do_not_unload=0, new_table=None):
    if action == 'get':
        try:
            t = tables[year, semester, ue]
            if t and do_not_unload:
                t.do_not_unload_add(do_not_unload)
            return t
        except KeyError:
            tables[year, semester, ue] = None
            return False
    elif action == 'del':
        try:            
            t = tables[year, semester, ue]
            if not t:
                return # Yet destroyed
            if t.do_not_unload:
                warn('Unload of do_not_unload: '
                     + str(t.ue) + repr(t.do_not_unload), what="warning")
                return False
            if t.active_pages:
                warn('Unload with active pages.', what="warning")
                return False
            # write access to the table will make an error.
            t.unloaded = True
            utilities.unload_module(t.module) # 2009-09-07 Add this
            del tables[year, semester, ue]            
            return True
        except KeyError:
            return # Yet destroyed
    elif action == 'replace':
        old = tables.get((year, semester, ue), None)
        if old:
            # XXX The module is unloaded just after the import.
            # Why is it necessary to do it again here?
            utilities.unload_module(old.module)
        tables[year, semester, ue] = new_table
        return new_table
    else:
        raise ValueError("Unknown action '%s'" % action)

def tables_values():
    # Use list because the tables may change while creating the list.
    # Do not use a tuple because it must be sortable
    # XXX Need a lock ?
    return [t
            for t in tuple(tables.values())
            if t not in (None, False, True)
            ]

            


def table(year, semester, ue, page=None, ticket=None, ro=False, create=True,
          do_not_unload=0):
    """With a ticket : return table and page
    Without ticket : return only the table
    """
    # utilities.warn('%s/%s/%s page=%s ticket=%s ro=%s create=%s' % (
    #    year, semester, ue, page, ticket, ro, create))
    if configuration.read_only:
        ro = True
    year = int(year)
    t = tables_manage('get', year, semester, ue, do_not_unload)
    if t is None:
        # Somebody else is creating the table
        while t is None:
            time.sleep(0.1)
            t = tables_manage('get', year, semester, ue, do_not_unload)

    if t is False:
        # I must create the table
        # Only one thread can be here at the same time.
        try:
            if ticket == None:
                if (not create
                    and not os.path.exists(table_filename(year, semester, ue))):
                    del tables[year, semester, ue]
                    return None
                t = Table(year, semester, ue, ro)
            else:
                t = Table(year, semester, ue, ro, user=ticket.user_name)
        except:
            # The table import failed, allow the new one to retry
            # tables[year, semester, ue] = False
            # raise
            utilities.send_backtrace("", "Can't load %s/%s/%s" % (
                    year, semester, ue))
            # tables[year, semester, ue] contains None
            # So no need to call table_manage
            del tables[year, semester, ue]
            if ticket:
                return None, None
            else:
                return None
        t.do_not_unload_add(do_not_unload) # Only on this case
        tables[year, semester, ue] = t
    else:
        # Table yet loaded
        t = t.update()

    t.atime = time.time()

    if page is not None and not t.modifiable and page >= len(t.pages):
        # Bug raised by TOMUSS restart (page list is lost)
        # XXX : if 2 users are on the same unmodifiable table,
        # then there is 50% chance that the second one is
        # with the bad page and so the bad ticket.
        # It will be accused of hacking.
        page = None
    
    if page == None:
        if ticket == None:
            return t
        if not t.readable_by(ticket):
            try:
                t.do_not_unload_remove('new_page')
            except ValueError:
                pass
            return None, None
        # The new page may append work to check_students_in_tables
        # And the work is done without call to 'table'.
        # So possible simultaneous execution is possible.
        t.lock()
        try:
            page = t.new_page(ticket.ticket,
                              ticket.user_name,
                              ticket.user_ip,
                              ticket.user_browser,
                              time.strftime('%Y%m%d%H%M%S'))
        finally:
            t.unlock()
        return t, page

    page = t.pages[page]
    page.check_identity(ticket.ticket, ticket.user_name,
                        ticket.user_ip, ticket.user_browser)
    return t, page


def remove_unused_tables():
    """Do not call from a thread"""
    warn('start', what="table")
    now = time.time()
    for atable in tables_values():
        if atable.active_pages:
            continue
        if now - atable.atime < configuration.unload_interval:
            continue
        atable.unload()
            
    warn('stop', what="table")

# continuous update of students lists for all tables
def check_students_in_tables():
    while True:
        time.sleep(configuration.students_check_interval)
        for t in tables_values():
            update_students.append(t)
            if configuration.regtest_sync:
                time.sleep(0.1)
            else:
                time.sleep(10) # Avoid overload


# continuous update of students lists
update_students = []

@utilities.add_a_lock # The lock is here for regression tests
def check_new_students_real():
    try:
        while update_students:
            t = update_students.pop()
            t.do_not_unload_add('check_new_students_real')
            # XXX Not a good test, but a lock is overkill
            if t.unloaded:
                continue
            try:
                t.send_update(None, "<script>set_updating(1);</script>")
                utilities.bufferize_this_file(t.filename)
                warn('start update students of %s' % t.ue, what="table")

                if t.template and hasattr(t.template, 'check'):
                    t.template.check(t)
                warn('done %s' % t.ue, what="table")
                mails = inscrits.L_batch.mails(list(t.logins()) + t.authors())
                mails.update(t.mails)
                t.change_mails(mails)
                if t.modifiable:
                    if t.force_update or getattr(t, 'update_inscrits', True):
                        for a_column in t.columns:
                            a_column.type.update_all(t, a_column)
                    t.force_update = 0
                t.update_the_abjs()
            finally:
                t.do_not_unload_remove('check_new_students_real')
                utilities.bufferize_this_file(None)
                t.send_update(None, "<script>set_updating(0);</script>")
            
    except IndexError:
        if not configuration.regtest:
            raise # Real problem
        # XXX The regtest raise a bug, why ?

indexes_to_update = []

def update_index(login, action=None):
    if not login:
        return
    if utilities.safe(login) != login:
        return
    c = utilities.manage_key('LOGINS', os.path.join(login, 'index'))
    if c:
        c = eval(c)
    else:
        c = []
    if action is None:
        return c
    c = action(c) # Update index content
    utilities.manage_key('LOGINS', os.path.join(login, 'index'),
                         content = repr(c))
    

def check_indexes_to_update():
    while indexes_to_update:
        tabl, old, new = indexes_to_update.pop(0)
        def remove(x):
            try:
                x.remove((tabl.year, tabl.semester, tabl.ue))
            except ValueError:
                pass
            return x
        def append(x):
            x.append((tabl.year, tabl.semester, tabl.ue))
            return x
        update_index(old, remove)
        update_index(new, append)
                
def check_new_students():
    while True:
        if configuration.regtest_sync:
            time.sleep(0.001)
        else:
            time.sleep(1)

        check_indexes_to_update()
        check_new_students_real()

def login_list(page, name):
    # XXX Not very clean the : configuration.teachers[-1]
    t = list(inscrits.L_slow.firstname_or_surname_to_logins(
        name.replace('_',' '),
        base=configuration.teachers[-1],
        attributes = [configuration.attr_login,
                      configuration.attr_surname,
                      configuration.attr_firstname,
                      'cn']
        ))
    t.sort(key = lambda x: x[1] + x[2])
    if t:
        s = []
        for lo, surname, firstname, cn in t:
            s.append('[' + utilities.js(lo.encode('utf-8'))
                     + ',' + utilities.js(surname.upper().encode('utf-8'))
                     + ',' + utilities.js(firstname.title().encode('utf-8'))
                     + ',' + utilities.js(cn.encode('utf-8'))
                     + ']')
    else:
        s = []
    s = '<script>login_list(' + utilities.js(name) + \
        ',[' + ',\n'.join(s) + ']);</script>\n'
    sender.append(page.browser_file, s)
            

# Do the real job to answer to modification requests
# It is complicated because we may receive request in bad order.
# So we keep a list of requests and sort it.

def it_is_a_bad_request(request, page, tabl, output_file):
    if page.request > request:
        # An old request was given. Assume same answer XXX
        tabl.do_not_unload_remove('page_action')
        try:
            warn('Old request asked : %d in place of %d' % (
                request, page.request))
            output_file.write(files.files['ok.png'])
            output_file.close()
        except IOError:
            pass
        except:
            utilities.send_backtrace('Exception request %d' % request)
        return True
    if tabl.unloaded:
        # No sense to do the do_not_unload_add(-1)
        utilities.send_backtrace('Request on unloaded table ' + tabl.ue,
                                 exception=False)
        return True

def should_be_delayed(request, page, tabl, r, t):
    if tabl.the_lock.locked():
        return True
    if page.request < request:
        # Count the number of identical request
        n = 0
        for old in t[::-1]:
            if old[0] == r[0] and old[1] == r[1]:
                n += 1
            else:
                break
        if n > 12:
            # We received 12 times the same request. (6 is too small)
            # So, we hit a request-accounting bug.
            # For example: server restart and browser 'update_content'
            # request that is not stored in the table data file.
            utilities.send_backtrace('Bad Request Number (%s!=%s)' %
                                     (page.request, request), exception=False)
            # We fix the page request number
            # So we no more delay this request handling
            page.request = request
            sender.append(page.browser_file,
                          '<script>Alert("ERROR_server_bug");</script>')
        else:
            return True

def process_request(page, tabl, action, path):
    page.request += 1
    page.answer = 'bug.png'
    tabl.lock()
    try:
        if action.startswith('column_attr_'):
            page.answer = tabl.column_attr(page, path[0],
                                           action[12:], path[1])
        elif action.startswith('table_attr_'):
            page.answer = tabl.table_attr(page,
                                          action[11:], path[0])
        elif action == 'cell_change':
            col, lin, value  = path
            page.answer = tabl.cell_change(page, col, lin, value)
        elif action == 'comment_change':
            col, lin, value = path
            page.answer = tabl.comment_change(page, col, lin, value)
        elif action == 'column_delete':
            col = path[0]
            page.answer = tabl.column_delete(page, col)
        elif action == 'login_list':
            login_list(page, utilities.safe(path[0]))
            page.answer = 'ok.png'
        elif action == 'update_content':
            if tabl not in update_students:
                update_students.append(tabl)
            tabl.force_update = 1
            page.answer = 'ok.png'
        else:
            warn('BUG: %s' % str(path), what="error")
    finally:
        tabl.do_not_unload_remove('page_action')
        tabl.unlock()
    # We don't want asynchronous update when doing regtest
    if configuration.regtest_sync:
        update_students.append(tabl)
        check_new_students_real()

request_list = []
def check_requests():
    while True:
        time.sleep(0.1)
        my_request_list = list(request_list)
        if len(my_request_list) == 0:
            # To not flush buffers of currently student list updates
            # In fact the buffering only gain 10% in user time and is
            # a little less secure. So it is disabled (see YYY comments)
            continue
        my_request_list.sort()
        valid_request = []
        for r in my_request_list:
            page_id, request, page, action, path, output_file = r
            tabl = page.table
            warn('R=%d P=%d A=%s P=%s DNU=%s' % (
                    request, page_id, action, path, tabl.do_not_unload),
                 what="DNU")
            if it_is_a_bad_request(request, page, tabl, output_file):
                request_list.remove(r)
                continue
            # utilities.bufferize_this_file(tabl.filename) # YYY
            # It is only useful to compare to previous requests
            valid_request.append(r)
            if should_be_delayed(request, page, tabl, r, valid_request):
                # do not decrement do_not_unload
                # Keep in request_list
                continue
            # Remove from request list.
            # If there is an error, browser will send a new request
            request_list.remove(r)
            try:
                real_bug = True
                process_request(page, tabl, action, path)
            except:
                warn('bug raised', what="error")
                utilities.send_backtrace('check_requests: '+action+repr(path))
                page.answer = 'bug.png'
                real_bug = False
            try:
                warn('Send %s(%s) %s' % (output_file, output_file.closed,
                                         page.answer), what="table")
                if not output_file.closed:
                    output_file.write(files.files[page.answer])
                    output_file.close()
                sender.append(page.browser_file,
                              '<script>saved(%d);</script>\n' % request,
                              index=len(tabl.sent_to_browsers)+1)
                if page.answer == 'bug.png' and real_bug:
                    sender.append(page.browser_file,
                                  '<script>Alert("ERROR_server_bug");</script>')
            except socket.error:
                pass
        # utilities.bufferize_this_file(None) # YYY Flush buffers
                           

# continuous send of packets to check connections
from . import ticket
def check_down_connections():
    while True:
        # This is done here, because testing this once per minute is fine.
        # Force rewrite of modified files in TMP/version/file
        # These files can be used by a static file server
        for f in files.files.values():
            try:
                str(f)
            except OSError:
                pass

        time.sleep(configuration.check_down_connections_interval)
        for ttable in tables_values():
            ttable.send_update(None, '<script>connected();</script>',
                               store=False)
            for page in ttable.active_pages:
                if page.ticket not in ticket.tickets:
                    warn('%s ticked expired page=%s ticket=%s' % (
                        ttable.filename,
                        page.page_id,
                        page.ticket), what='error')
                    continue
                                                  
                t = ticket.tickets[page.ticket]
                if '-' in t.ticket:
                    sender.send_live_status('<script>d("%s","%s","%s",0.01,"pong", "%s",%s,%s);</script>\n'%(
                        t.user_ip,
                        t.ticket.split('-')[1] + '/' + t.user_name,
                        t.access_right(),
                        ttable.year, utilities.js(ttable.semester),
                        utilities.js(ttable.ue)))

                    
# Update computed values

cell_changed_list_fast = []

def update_computed_values_fast():
    while True:
        time.sleep(0.1)
        while cell_changed_list_fast:
            the_table, lin, a_column = cell_changed_list_fast.pop()
            for col in the_table.columns.use(a_column):
                col.type.update_one(the_table, lin, col)
            # Update mail if new teacher
            login = the_table.lines[lin][a_column.data_col].author
            if login not in the_table.mails:
                m = inscrits.L_fast.mail(login)
                if m:
                    the_table.update_mail(login, m.encode('utf-8'))
            # Update mail if login changed
            if a_column.data_col == 0:
                login = the_table.lines[lin][0].value
                if isinstance(login, str):
                    m = inscrits.L_fast.mail(login)
                    if m:
                        the_table.update_mail(login, m)

column_changed_list = []

def update_computed_values_slow():
    while True:
        time.sleep(0.1)
        while column_changed_list:
            the_table, a_column, attr = column_changed_list.pop()
            a_column.type.update_all(the_table, a_column, attr)
            for col in the_table.columns.use(a_column):
                col.type.update_all(the_table, col)

def virtual_table(server, the_columns, the_lines, table_attrs={}, js=""):
    """Send the table to the browser without storage.
    Do not use in a not threaded plugin.
    """
    if not hasattr(server, "year"):
        server.year = server.__dict__.get("the_year", 0)
    if not hasattr(server, "semester"):
        if hasattr(server, 'the_semester'):
            server.semester = server.the_semester
        else:
            server.semester = "?"
    for i, a_column in enumerate(the_columns):
        if not a_column.position:
            a_column.position = i
    class TMP:
        pass
    tmp = TMP()
    tmp.__dict__ = table_attrs
    
    server.the_file.write(table_head(server.year,
                                     server.semester,
                                     server.ticket.ticket,
                                     create_pref = False,
                                     attrs_from=tmp,
                                     user_name=server.ticket.user_name
                                     )
                          )
    lines = []
    for i, line in enumerate(the_lines):
        lines.append('"%d": %s' % (i, line.js()))
    lines = '{' + ',\n'.join(lines) + '}'
    columns = '[' + ',\n'.join([the_column.js(hide=False)
                                for the_column in the_columns]) + ']'
    server.the_file.write("""
    <script>
    %s
    %s
    function initialize()
    {
    if ( ! wait_scripts("initialize()") )
               return ;
    document.write(head_html()) ;
    insert_middle();
    columns = %s ;
    lines = %s ;
    document.write(tail_html()) ;
    table_attr.table_title = %s ;
    runlog(columns, lines) ;
    }
    initialize() ;
    </script>
    """ % (utilities.wait_scripts(),
           js, columns, lines, utilities.js(repr(server.the_path)) ))

    logins = list(line[0].value
                  for line in the_lines
                  if line[0].value
                  )
    mails = inscrits.L_batch.mails(logins)
    server.the_file.write("<script>Xtable_attr('mails',%s);</script>\n"
                          % utilities.js(mails))
    server.close_connection_now()
       
def start_threads():
    utilities.start_new_thread_immortal(check_new_students, ())
    utilities.start_new_thread_immortal(check_students_in_tables, ())
    utilities.start_new_thread_immortal(check_requests, ())
    utilities.start_new_thread_immortal(check_down_connections, ())
    utilities.start_new_thread_immortal(update_computed_values_fast, ())
    utilities.start_new_thread_immortal(update_computed_values_slow, ())