[API/Mobidziennik] Implement syncing extra lessons. (#83)

app/src/main/java/pl/szczodrzynski/edziennik/data/api/edziennik/mobidziennik/data/api/MobidziennikApiTeams.kt

@@ -35,7 +35,6 @@ class MobidziennikApiTeams(val data: DataMobidziennik, tableTeams: List<String>?
         }
         if (tableRelations != null) {
             val allTeams = data.teamList.values()
-            data.teamList.clear()
 
             for (row in tableRelations) {
                 if (row.isEmpty())

app/src/main/java/pl/szczodrzynski/edziennik/data/api/edziennik/mobidziennik/data/api/MobidziennikApiTimetable.kt

@@ -24,7 +24,7 @@ class MobidziennikApiTimetable(val data: DataMobidziennik, rows: List<String>) {
         val dataStart = Date.getToday()
         val dataEnd = dataStart.clone().stepForward(0, 0, 7 + (6 - dataStart.weekDay))
 
-        data.toRemove.add(DataRemoveModel.Timetable.between(dataStart.clone(), dataEnd))
+        data.toRemove.add(DataRemoveModel.Timetable.between(dataStart.clone(), dataEnd, isExtra = false))
 
         val dataDays = mutableListOf<Int>()
         while (dataStart <= dataEnd) {

app/src/main/java/pl/szczodrzynski/edziennik/data/api/edziennik/mobidziennik/data/web/MobidziennikWebTimetable.kt

@@ -11,6 +11,7 @@ import pl.szczodrzynski.edziennik.data.api.Regexes
 import pl.szczodrzynski.edziennik.data.api.edziennik.mobidziennik.DataMobidziennik
 import pl.szczodrzynski.edziennik.data.api.edziennik.mobidziennik.ENDPOINT_MOBIDZIENNIK_WEB_TIMETABLE
 import pl.szczodrzynski.edziennik.data.api.edziennik.mobidziennik.data.MobidziennikWeb
+import pl.szczodrzynski.edziennik.data.api.models.DataRemoveModel.Timetable.Companion.between
 import pl.szczodrzynski.edziennik.data.db.entity.Lesson
 import pl.szczodrzynski.edziennik.data.db.entity.Metadata
 import pl.szczodrzynski.edziennik.data.db.entity.SYNC_ALWAYS
@@ -81,22 +82,30 @@ class MobidziennikWebTimetable(
         } ?: currentWeekStart
 
         val syncFutureDate = startDate > nextWeekEnd
-        // TODO: 2021-09-09 make DataRemoveModel keep extra lessons
+        val syncPastDate = startDate < currentWeekStart
-        val syncExtraLessons = false && System.currentTimeMillis() - (lastSync ?: 0) > 2 * DAY * MS
+        val syncExtraLessons = System.currentTimeMillis() - (lastSync ?: 0) > 2 * DAY * MS
-        if (!syncFutureDate && !syncExtraLessons) {
+        // sync not needed - everything present in the "API"
+        if (!syncFutureDate && !syncPastDate && !syncExtraLessons) {
             onSuccess(ENDPOINT_MOBIDZIENNIK_WEB_TIMETABLE)
         }
         else {
             val types = when {
-                syncFutureDate -> mutableListOf("podstawowy")//, "pozalekcyjny")
+                syncFutureDate || syncPastDate -> mutableListOf("podstawowy", "pozalekcyjny")
                 syncExtraLessons -> mutableListOf("pozalekcyjny")
                 else -> mutableListOf()
             }
 
+            val syncingExtra = types.contains("pozalekcyjny")
+
             syncTypes(types, startDate) {
-                // set as synced now only when not syncing future date
+                if (syncingExtra) {
-                // (to avoid waiting 2 days for normal sync after future sync)
+                    val endDate = startDate.clone().stepForward(0, 0, 7)
-                if (syncExtraLessons)
+                    data.toRemove.add(between(startDate, endDate, isExtra = true))
+                }
+
+                // set as synced now only when not syncing future/past date
+                // (to avoid waiting 2 days for normal sync after future/past sync)
+                if (!syncFutureDate && !syncPastDate)
                     data.setSyncNext(ENDPOINT_MOBIDZIENNIK_WEB_TIMETABLE, SYNC_ALWAYS)
                 onSuccess(ENDPOINT_MOBIDZIENNIK_WEB_TIMETABLE)
             }
@@ -113,7 +122,7 @@ class MobidziennikWebTimetable(
             MobidziennikLuckyNumberExtractor(data, html)
             readRangesH(html)
             readRangesV(html)
-            readLessons(html)
+            readLessons(html, isExtra = type == "pozalekcyjny")
             syncTypes(types, startDate, onSuccess)
         }
     }
@@ -183,7 +192,7 @@ class MobidziennikWebTimetable(
     }
 
     @SuppressLint("LongLogTag", "LogNotTimber")
-    private fun readLessons(html: String) {
+    private fun readLessons(html: String, isExtra: Boolean) {
         val matches = Regexes.MOBIDZIENNIK_TIMETABLE_CELL.findAll(html)
 
         val noLessonDays = mutableListOf<Date>()
@@ -215,51 +224,63 @@ class MobidziennikWebTimetable(
             var teamName: String? = null
             val items = (cleanup(match[3]) + cleanup(match[4])).toMutableList()
 
+            // comparing items size before and after the iteration
             var length = 0
             while (items.isNotEmpty() && length != items.size) {
                 length = items.size
-                val toRemove = mutableListOf<String?>()
+                var i = 0
-                items.forEachIndexed { i, item ->
+                while (i < items.size) {
+                    // just to remain safe - I have no idea how all of this works.
+                    if (i < 0)
+                        break
+                    val item = items[i]
                     when {
-                        item.isEmpty() ->
+                        // remove empty items
-                            toRemove.add(item)
+                        item.isEmpty() -> {
-                        item.contains(":") && item.contains(" - ") ->
+                            items.remove(item)
-                            toRemove.add(item)
+                            i--
+                        }
+                        // remove HH:MM items - it's calculated from the block position
+                        item.contains(":") && item.contains(" - ") -> {
+                            items.remove(item)
+                            i--
+                        }
 
                         item.startsWith("%") -> {
-                            subjectName = item.trim('%')
+                            // the one wrapped in % is the short subject name
-                            // I have no idea what's going on here
+                            items.remove(item)
-                            // ok now seriously.. the subject (long or short) item
+                            // remove the first remaining item
-                            // may NOT be 0th, as the HH:MM - HH:MM item may be before
+                            subjectName = items.removeAt(0)
-                            // or even the typeName item. As these are always **before**,
+                            // decrement the index counter
-                            // they are removed in previous iterations, so the first not removed
+                            i -= 2
-                            // item should be the long/short subjectName needing to be removed now.
-                            toRemove.add(items[toRemove.size])
-                            // ...and this has to be added later
-                            toRemove.add(item)
                         }
 
-                        item.startsWith("&") -> {
+                        item.startsWith("$") -> {
-                            typeName = item.trim('&')
+                            typeName = item.trim('$')
-                            toRemove.add(item)
+                            items.remove(item)
+                            i--
                         }
-                        typeName != null && (item.contains(typeName!!) || item.contains("</small>")) -> {
+                        typeName != null && (item.contains(typeName) || item.contains("</small>")) -> {
-                            toRemove.add(item)
+                            items.remove(item)
+                            i--
                         }
 
                         item.contains("(") && item.contains(")") -> {
                             classroomName = classroomRegex.find(item)?.get(1)
                             items[i] = item.replace("($classroomName)", "").trim()
                         }
-                        classroomName != null && item.contains(classroomName!!) -> {
+                        classroomName != null && item.contains(classroomName) -> {
                             items[i] = item.replace("($classroomName)", "").trim()
                         }
 
-                        item.contains("class=\"wyjatek tooltip\"") ->
+                        item.contains("class=\"wyjatek tooltip\"") -> {
-                            toRemove.add(item)
+                            items.remove(item)
+                            i--
+                        }
                     }
+                    // finally advance to the next item
+                    i++
                 }
-                items.removeAll(toRemove)
             }
 
             if (items.size == 2 && items[0].contains(" - ")) {
@@ -311,6 +332,7 @@ class MobidziennikWebTimetable(
                 }
 
                 it.id = it.buildId()
+                it.isExtra = isExtra
 
                 val seen = profile?.empty == false || lessonDate < Date.getToday()
 

app/src/main/java/pl/szczodrzynski/edziennik/data/api/models/DataRemoveModel.kt

@@ -11,19 +11,19 @@ import pl.szczodrzynski.edziennik.data.db.dao.TimetableDao
 import pl.szczodrzynski.edziennik.utils.models.Date
 
 open class DataRemoveModel {
-    data class Timetable(private val dateFrom: Date?, private val dateTo: Date?) : DataRemoveModel() {
+    data class Timetable(private val dateFrom: Date?, private val dateTo: Date?, private val isExtra: Boolean?) : DataRemoveModel() {
         companion object {
-            fun from(dateFrom: Date) = Timetable(dateFrom, null)
+            fun from(dateFrom: Date, isExtra: Boolean? = null) = Timetable(dateFrom, null, isExtra)
-            fun to(dateTo: Date) = Timetable(null, dateTo)
+            fun to(dateTo: Date, isExtra: Boolean? = null) = Timetable(null, dateTo, isExtra)
-            fun between(dateFrom: Date, dateTo: Date) = Timetable(dateFrom, dateTo)
+            fun between(dateFrom: Date, dateTo: Date, isExtra: Boolean? = null) = Timetable(dateFrom, dateTo, isExtra)
         }
 
         fun commit(profileId: Int, dao: TimetableDao) {
             if (dateFrom != null && dateTo != null) {
-                dao.dontKeepBetweenDates(profileId, dateFrom, dateTo)
+                dao.dontKeepBetweenDates(profileId, dateFrom, dateTo, isExtra ?: false)
             } else {
-                dateFrom?.let { dateFrom -> dao.dontKeepFromDate(profileId, dateFrom) }
+                dateFrom?.let { dateFrom -> dao.dontKeepFromDate(profileId, dateFrom, isExtra ?: false) }
-                dateTo?.let { dateTo -> dao.dontKeepToDate(profileId, dateTo) }
+                dateTo?.let { dateTo -> dao.dontKeepToDate(profileId, dateTo, isExtra ?: false) }
             }
         }
     }

app/src/main/java/pl/szczodrzynski/edziennik/data/db/AppDb.kt

@@ -43,7 +43,7 @@ import pl.szczodrzynski.edziennik.data.db.migration.*
     LibrusLesson::class,
     TimetableManual::class,
     Metadata::class
-], version = 94)
+], version = 95)
 @TypeConverters(
         ConverterTime::class,
         ConverterDate::class,
@@ -179,7 +179,8 @@ abstract class AppDb : RoomDatabase() {
                 Migration91(),
                 Migration92(),
                 Migration93(),
-                Migration94()
+                Migration94(),
+                Migration95(),
         ).allowMainThreadQueries().build()
     }
 }

app/src/main/java/pl/szczodrzynski/edziennik/data/db/dao/TimetableDao.kt

@@ -107,12 +107,12 @@ abstract class TimetableDao : BaseDao<Lesson, LessonFull> {
     fun getByIdNow(profileId: Int, id: Long) =
             getOneNow("$QUERY WHERE timetable.profileId = $profileId AND timetable.id = $id")
 
-    @Query("UPDATE timetable SET keep = 0 WHERE profileId = :profileId AND type != -1 AND ((type != 3 AND date >= :dateFrom) OR ((type = 3 OR type = 1) AND oldDate >= :dateFrom))")
+    @Query("UPDATE timetable SET keep = 0 WHERE profileId = :profileId AND isExtra = :isExtra AND type != -1 AND ((type != 3 AND date >= :dateFrom) OR ((type = 3 OR type = 1) AND oldDate >= :dateFrom))")
-    abstract fun dontKeepFromDate(profileId: Int, dateFrom: Date)
+    abstract fun dontKeepFromDate(profileId: Int, dateFrom: Date, isExtra: Boolean)
 
-    @Query("UPDATE timetable SET keep = 0 WHERE profileId = :profileId AND type != -1 AND ((type != 3 AND date <= :dateTo) OR ((type = 3 OR type = 1) AND oldDate <= :dateTo))")
+    @Query("UPDATE timetable SET keep = 0 WHERE profileId = :profileId AND isExtra = :isExtra AND type != -1 AND ((type != 3 AND date <= :dateTo) OR ((type = 3 OR type = 1) AND oldDate <= :dateTo))")
-    abstract fun dontKeepToDate(profileId: Int, dateTo: Date)
+    abstract fun dontKeepToDate(profileId: Int, dateTo: Date, isExtra: Boolean)
 
-    @Query("UPDATE timetable SET keep = 0 WHERE profileId = :profileId AND type != -1 AND ((type != 3 AND date >= :dateFrom AND date <= :dateTo) OR ((type = 3 OR type = 1) AND oldDate >= :dateFrom AND oldDate <= :dateTo))")
+    @Query("UPDATE timetable SET keep = 0 WHERE profileId = :profileId AND isExtra = :isExtra AND type != -1 AND ((type != 3 AND date >= :dateFrom AND date <= :dateTo) OR ((type = 3 OR type = 1) AND oldDate >= :dateFrom AND oldDate <= :dateTo))")
-    abstract fun dontKeepBetweenDates(profileId: Int, dateFrom: Date, dateTo: Date)
+    abstract fun dontKeepBetweenDates(profileId: Int, dateFrom: Date, dateTo: Date, isExtra: Boolean)
 }

app/src/main/java/pl/szczodrzynski/edziennik/data/db/entity/Lesson.kt

@@ -48,6 +48,8 @@ open class Lesson(
     var oldTeamId: Long? = null
     var oldClassroom: String? = null
 
+    var isExtra: Boolean = false
+
     val displayDate: Date?
         get() {
             if (type == TYPE_SHIFTED_SOURCE)
@@ -121,11 +123,13 @@ open class Lesson(
         return true
     }
 
-    override fun hashCode(): Int { // intentionally ignoring ID and display* here
+    override fun hashCode(): Int { // intentionally ignoring ID, display* and isExtra here
         var result = profileId
         result = 31 * result + type
         result = 31 * result + (date?.hashCode() ?: 0)
-        result = 31 * result + (lessonNumber ?: 0)
+        // this creates problems in Mobidziennik with extra lessons
+        // ... and is not generally useful anyway
+        // result = 31 * result + (lessonNumber ?: 0)
         result = 31 * result + (startTime?.hashCode() ?: 0)
         result = 31 * result + (endTime?.hashCode() ?: 0)
         result = 31 * result + (subjectId?.hashCode() ?: 0)
@@ -133,7 +137,7 @@ open class Lesson(
         result = 31 * result + (teamId?.hashCode() ?: 0)
         result = 31 * result + (classroom?.hashCode() ?: 0)
         result = 31 * result + (oldDate?.hashCode() ?: 0)
-        result = 31 * result + (oldLessonNumber ?: 0)
+        // result = 31 * result + (oldLessonNumber ?: 0)
         result = 31 * result + (oldStartTime?.hashCode() ?: 0)
         result = 31 * result + (oldEndTime?.hashCode() ?: 0)
         result = 31 * result + (oldSubjectId?.hashCode() ?: 0)

app/src/main/java/pl/szczodrzynski/edziennik/data/db/migration/Migration95.kt

@@ -0,0 +1,15 @@
+/*
+ * Copyright (c) Kuba Szczodrzyński 2021-10-1.
+ */
+
+package pl.szczodrzynski.edziennik.data.db.migration
+
+import androidx.room.migration.Migration
+import androidx.sqlite.db.SupportSQLiteDatabase
+
+class Migration95 : Migration(94, 95) {
+    override fun migrate(database: SupportSQLiteDatabase) {
+        // timetable - is extra flag
+        database.execSQL("ALTER TABLE timetable ADD COLUMN isExtra INT NOT NULL DEFAULT 0;")
+    }
+}