From 63e1d5b283c347f51951569e83ce552a3dabd1e5 Mon Sep 17 00:00:00 2001 From: moonleay Date: Sun, 15 Oct 2023 15:17:15 +0200 Subject: [PATCH 01/10] feat!: added subscriptionTimestamp to SubscriptionsTable --- .../rssbot/data/database/entry/SubscriptionData.kt | 1 + .../data/database/repository/SubscriptionRepository.kt | 10 +++++++--- .../rssbot/data/database/tables/SubscriptionsTable.kt | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/net/moonleay/rssbot/data/database/entry/SubscriptionData.kt b/src/main/kotlin/net/moonleay/rssbot/data/database/entry/SubscriptionData.kt index 71f47b5..3896b37 100644 --- a/src/main/kotlin/net/moonleay/rssbot/data/database/entry/SubscriptionData.kt +++ b/src/main/kotlin/net/moonleay/rssbot/data/database/entry/SubscriptionData.kt @@ -27,4 +27,5 @@ data class SubscriptionData( val feedColor: FeedColor, val feedName: String, val feedUrl: String, + val subscriptionTimestamp: Long, ) diff --git a/src/main/kotlin/net/moonleay/rssbot/data/database/repository/SubscriptionRepository.kt b/src/main/kotlin/net/moonleay/rssbot/data/database/repository/SubscriptionRepository.kt index f80d96b..95cc5fb 100644 --- a/src/main/kotlin/net/moonleay/rssbot/data/database/repository/SubscriptionRepository.kt +++ b/src/main/kotlin/net/moonleay/rssbot/data/database/repository/SubscriptionRepository.kt @@ -38,7 +38,8 @@ object SubscriptionRepository { it[SubscriptionsTable.channelId], EmbedUtil.getColorFromString(it[SubscriptionsTable.subscriptionColor]), it[SubscriptionsTable.subscriptionName], - it[SubscriptionsTable.feedUrl] + it[SubscriptionsTable.feedUrl], + it[SubscriptionsTable.subscriptionTimestamp] ) ) } @@ -59,7 +60,8 @@ object SubscriptionRepository { it[SubscriptionsTable.channelId], EmbedUtil.getColorFromString(it[SubscriptionsTable.subscriptionColor]), it[SubscriptionsTable.subscriptionName], - it[SubscriptionsTable.feedUrl] + it[SubscriptionsTable.feedUrl], + it[SubscriptionsTable.subscriptionTimestamp] ) ) } @@ -80,7 +82,8 @@ object SubscriptionRepository { it[SubscriptionsTable.channelId], EmbedUtil.getColorFromString(it[SubscriptionsTable.subscriptionColor]), it[SubscriptionsTable.subscriptionName], - it[SubscriptionsTable.feedUrl] + it[SubscriptionsTable.feedUrl], + it[SubscriptionsTable.subscriptionTimestamp] ) } } @@ -115,6 +118,7 @@ object SubscriptionRepository { it[subscriptionColor] = data.feedColor.readableName it[subscriptionName] = data.feedName it[feedUrl] = data.feedUrl + it[subscriptionTimestamp] = data.subscriptionTimestamp } } diff --git a/src/main/kotlin/net/moonleay/rssbot/data/database/tables/SubscriptionsTable.kt b/src/main/kotlin/net/moonleay/rssbot/data/database/tables/SubscriptionsTable.kt index de0ce39..ada2744 100644 --- a/src/main/kotlin/net/moonleay/rssbot/data/database/tables/SubscriptionsTable.kt +++ b/src/main/kotlin/net/moonleay/rssbot/data/database/tables/SubscriptionsTable.kt @@ -28,4 +28,5 @@ object SubscriptionsTable : Table(name = "subscriptions") { val subscriptionColor = text("subscription_color") val subscriptionName = text("subscription_name") val feedUrl = text("feed_url") + val subscriptionTimestamp = long("subscription_timestamp") } From 39802598b2bc032accd968af1a4c36b079cb034c Mon Sep 17 00:00:00 2001 From: moonleay Date: Sun, 15 Oct 2023 15:17:53 +0200 Subject: [PATCH 02/10] feat: added CleanUpExtension --- .../rssbot/extensions/CleanUpExtension.kt | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 src/main/kotlin/net/moonleay/rssbot/extensions/CleanUpExtension.kt diff --git a/src/main/kotlin/net/moonleay/rssbot/extensions/CleanUpExtension.kt b/src/main/kotlin/net/moonleay/rssbot/extensions/CleanUpExtension.kt new file mode 100644 index 0000000..f2abff6 --- /dev/null +++ b/src/main/kotlin/net/moonleay/rssbot/extensions/CleanUpExtension.kt @@ -0,0 +1,39 @@ +package net.moonleay.rssbot.extensions + +import com.kotlindiscord.kord.extensions.extensions.Extension +import com.kotlindiscord.kord.extensions.extensions.ephemeralSlashCommand +import com.kotlindiscord.kord.extensions.types.respond +import kotlinx.coroutines.flow.filter +import net.moonleay.rssbot.Bot +import net.moonleay.rssbot.data.database.repository.SubscriptionRepository +import net.moonleay.rssbot.util.EmbedColor +import net.moonleay.rssbot.util.MessageUtil + +class CleanUpExtension : Extension() { + + override val name = "cleanup" + override val allowApplicationCommandInDMs: Boolean + get() = false + + + override suspend fun setup() { + ephemeralSlashCommand() { + name = "cleanup" + description = "Clean all messages from this bot in this channel" + this.action { + val user = this.user.asUser() + this.channel.messages.filter { it.author?.id == Bot.bot.kordRef.selfId }.collect { it.delete() } + this.respond { + this.embeds.add( + MessageUtil.getEmbed( + EmbedColor.INFO, + "Cleaned up", + "Cleaned up all messages from this bot in this channel", + user.username + ) + ) + } + } + } + } +} From f217e4ff0711bb6ef1509fa180f1d08f53eaca52 Mon Sep 17 00:00:00 2001 From: moonleay Date: Sun, 15 Oct 2023 15:18:02 +0200 Subject: [PATCH 03/10] feat: added SendFeedExtension --- .../rssbot/extensions/SendFeedExtension.kt | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 src/main/kotlin/net/moonleay/rssbot/extensions/SendFeedExtension.kt diff --git a/src/main/kotlin/net/moonleay/rssbot/extensions/SendFeedExtension.kt b/src/main/kotlin/net/moonleay/rssbot/extensions/SendFeedExtension.kt new file mode 100644 index 0000000..11cf8d9 --- /dev/null +++ b/src/main/kotlin/net/moonleay/rssbot/extensions/SendFeedExtension.kt @@ -0,0 +1,113 @@ +/* + * RSSBot + * Copyright (C) 2023 moonleay + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package net.moonleay.rssbot.extensions + +import com.kotlindiscord.kord.extensions.commands.Arguments +import com.kotlindiscord.kord.extensions.commands.converters.impl.string +import com.kotlindiscord.kord.extensions.extensions.Extension +import com.kotlindiscord.kord.extensions.extensions.ephemeralSlashCommand +import com.kotlindiscord.kord.extensions.types.respond +import com.overzealous.remark.Options +import com.overzealous.remark.Remark +import com.prof18.rssparser.RssParser +import com.prof18.rssparser.model.RssChannel +import com.prof18.rssparser.model.RssItem +import dev.kord.rest.builder.message.EmbedBuilder.Limits.title +import net.moonleay.rssbot.data.database.repository.RSSRepository +import net.moonleay.rssbot.data.database.repository.SubscriptionRepository +import net.moonleay.rssbot.extensions.components.FeedColor +import net.moonleay.rssbot.util.EmbedColor +import net.moonleay.rssbot.util.EmbedUtil +import net.moonleay.rssbot.util.MessageUtil + +class SendFeedExtension : Extension() { + + override val name = "sendfeed" + override val allowApplicationCommandInDMs: Boolean + get() = false + + + override suspend fun setup() { + ephemeralSlashCommand(::StuffArguments) { + name = "sendfeed" + description = "Send a feed to this channel" + this.action { + val feedUrl = arguments.feedUrl + val user = this.user.asUser() + if (feedUrl == null) { + this.respond { + this.embeds.add( + MessageUtil.getEmbed( + EmbedColor.ERROR, + "Missing Arguments", + "You are missing one or more arguments", + user.username + ) + ) + } + return@action + } + val parser = RssParser() + var rss: RssChannel? = null + runCatching { // this sucks + rss = parser.getRssChannel(feedUrl) + }.onFailure { + this.respond { + this.embeds.add( + MessageUtil.getEmbed( + EmbedColor.ERROR, + "Invalid Feed", + "The feed you provided is invalid", + user.username + ) + ) + } + return@action + } + if (rss == null) + return@action + rss!!.items.forEach { + this.respond { + this.embeds.add( + MessageUtil.getRSSEmbed( + FeedColor.GREEN, + it.author ?: "Anonymous", + it.title ?: "Untitled", + if (it.description == null) "No description" else Remark(EmbedUtil.getHTMLtoMarkdownOptions()).convertFragment(it.description!!), + it.link ?: "https://moonleay.net/", + it.image ?: "", + if (it.link != null) EmbedUtil.getSiteLogo(it.link!!) else "https://static.moonleay.net/img/no-image.png", + "Manual Feed", + it.pubDate?: "unknown" + ) + ) + } + } + + } + } + } + + inner class StuffArguments : Arguments() { + val feedUrl by string { + this.name = "url" + this.description = "The url of the feed to send" + } + } +} From 9468efa10dec32e96a7535afd893b0840e236b36 Mon Sep 17 00:00:00 2001 From: moonleay Date: Sun, 15 Oct 2023 15:22:32 +0200 Subject: [PATCH 04/10] fix: added Subscription timestamp --- .../kotlin/net/moonleay/rssbot/extensions/FeedExtension.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/net/moonleay/rssbot/extensions/FeedExtension.kt b/src/main/kotlin/net/moonleay/rssbot/extensions/FeedExtension.kt index 06b40d1..dcfbe81 100644 --- a/src/main/kotlin/net/moonleay/rssbot/extensions/FeedExtension.kt +++ b/src/main/kotlin/net/moonleay/rssbot/extensions/FeedExtension.kt @@ -106,7 +106,8 @@ class FeedExtension : Extension() { this.channel.id.value.toLong(), feedColor, feedName, - feedUrl + feedUrl, + System.currentTimeMillis() ) ) Logger.out("added feed as id ${id.resultedValues!!.first()[SubscriptionsTable.id]}") From e4a593b7d37d0740be04b1e1a750393354000256 Mon Sep 17 00:00:00 2001 From: moonleay Date: Sun, 15 Oct 2023 15:23:41 +0200 Subject: [PATCH 05/10] feat: added getHTMLtoMarkdownOptions func Signed-off-by: moonleay --- src/main/kotlin/net/moonleay/rssbot/util/EmbedUtil.kt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/kotlin/net/moonleay/rssbot/util/EmbedUtil.kt b/src/main/kotlin/net/moonleay/rssbot/util/EmbedUtil.kt index 6ce00ca..634d3ce 100644 --- a/src/main/kotlin/net/moonleay/rssbot/util/EmbedUtil.kt +++ b/src/main/kotlin/net/moonleay/rssbot/util/EmbedUtil.kt @@ -18,6 +18,8 @@ package net.moonleay.rssbot.util +import com.overzealous.remark.IgnoredHtmlElement +import com.overzealous.remark.Options import dev.kord.common.entity.ButtonStyle import dev.kord.core.entity.Embed import dev.kord.rest.builder.component.ActionRowBuilder @@ -141,4 +143,11 @@ object EmbedUtil { return "https://www.google.com/s2/favicons?domain=${str.split("/")[2]}" } + fun getHTMLtoMarkdownOptions(): Options { + val o = Options() + o.ignoredHtmlElements.add(IgnoredHtmlElement("img")) + o.inlineLinks = true + return o + } + } From 0c7f51ba726af1f9caa20301c4bf2ae3ab9ece0b Mon Sep 17 00:00:00 2001 From: moonleay Date: Sun, 15 Oct 2023 15:27:20 +0200 Subject: [PATCH 06/10] feat: improved RSSEmbed Signed-off-by: moonleay --- src/main/kotlin/net/moonleay/rssbot/util/MessageUtil.kt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/net/moonleay/rssbot/util/MessageUtil.kt b/src/main/kotlin/net/moonleay/rssbot/util/MessageUtil.kt index d6d8ace..e01058b 100644 --- a/src/main/kotlin/net/moonleay/rssbot/util/MessageUtil.kt +++ b/src/main/kotlin/net/moonleay/rssbot/util/MessageUtil.kt @@ -174,11 +174,12 @@ object MessageUtil { imageUrl: String, logo: String, feedName: String, + published: String, ): EmbedBuilder { val ebb = EmbedBuilder() val now: LocalDateTime = LocalDateTime.now() - if (title.length > 250) - ebb.title = title.substring(0, 250) + "[...]" + if (title.length > 25) + ebb.title = title.substring(0, 25) + "..." else ebb.title = title ebb.author { @@ -188,7 +189,7 @@ object MessageUtil { ebb.description = "${if (description != "No description") description else ""}\n[[open full article]]($url)" ebb.color = color.color.color ebb.footer { - this.text = "> rssbot, ($feedName) / sent at ${dtf.format(now)}" + this.text = "> rssbot, ($feedName) \n/ pub $published \n/ rel ${dtf.format(now)}" this.icon = "https://static.moonleay.net/img/rss.png" } ebb.thumbnail { From d60b7ba92f762ba3c46066f3aaaf1533401881f4 Mon Sep 17 00:00:00 2001 From: moonleay Date: Sun, 15 Oct 2023 15:30:37 +0200 Subject: [PATCH 07/10] feat!: removed unused funcs, added getUnixTimeFromStamp Signed-off-by: moonleay --- .../net/moonleay/rssbot/util/TimeUtil.kt | 42 ++----------------- 1 file changed, 4 insertions(+), 38 deletions(-) diff --git a/src/main/kotlin/net/moonleay/rssbot/util/TimeUtil.kt b/src/main/kotlin/net/moonleay/rssbot/util/TimeUtil.kt index 89ff4b8..416d3a6 100644 --- a/src/main/kotlin/net/moonleay/rssbot/util/TimeUtil.kt +++ b/src/main/kotlin/net/moonleay/rssbot/util/TimeUtil.kt @@ -18,47 +18,13 @@ package net.moonleay.rssbot.util -import kotlinx.datetime.DayOfWeek import java.time.ZonedDateTime -import java.util.concurrent.TimeUnit +import java.time.format.DateTimeFormatter object TimeUtil { - - fun getTimeFormatedShortend(time2: Long, showS: Boolean): String { - var time = time2 - val days: Long = TimeUnit.MILLISECONDS - .toDays(time) - time -= TimeUnit.DAYS.toMillis(days) - val hours: Long = TimeUnit.MILLISECONDS - .toHours(time) - time -= TimeUnit.HOURS.toMillis(hours) - val minutes: Long = TimeUnit.MILLISECONDS - .toMinutes(time) - time -= TimeUnit.MINUTES.toMillis(minutes) - val seconds: Long = TimeUnit.MILLISECONDS - .toSeconds(time) - var s = "" - if (days >= 1) { - s += days.toString() + "d " - } - if (hours >= 1) { - s += hours.toString() + "h " - } - if (minutes >= 1) { - s += minutes.toString() + "m " - } - if (seconds >= 1 && hours < 1 && showS) { - s += seconds.toString() + "s" - } - if (s.isEmpty() || s.isBlank()) { - s = "None" - } - return s - } - - // Returns the day of the month of the monday of this week - fun getMondayDayOfMonth(): Int { - return ZonedDateTime.now().with(DayOfWeek.MONDAY).dayOfMonth + fun getUnixTimeFromStamp(input: String): Long { // Pattern: Sun, 15 Oct 2023 11:04:57 GMT + val formatter = DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss z") + return ZonedDateTime.parse(input, formatter).toEpochSecond() * 1000 } } From 0e2e3c802281d63bfa112a672154764307790a83 Mon Sep 17 00:00:00 2001 From: moonleay Date: Sun, 15 Oct 2023 15:46:04 +0200 Subject: [PATCH 08/10] feat: improved FeedUpdater message Signed-off-by: moonleay --- .../net/moonleay/rssbot/jobs/FeedUpdater.kt | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/net/moonleay/rssbot/jobs/FeedUpdater.kt b/src/main/kotlin/net/moonleay/rssbot/jobs/FeedUpdater.kt index ff712b6..810175c 100644 --- a/src/main/kotlin/net/moonleay/rssbot/jobs/FeedUpdater.kt +++ b/src/main/kotlin/net/moonleay/rssbot/jobs/FeedUpdater.kt @@ -18,6 +18,7 @@ package net.moonleay.rssbot.jobs +import com.overzealous.remark.Remark import com.prof18.rssparser.RssParser import com.prof18.rssparser.model.RssChannel import dev.inmo.krontab.KronScheduler @@ -35,6 +36,7 @@ import net.moonleay.rssbot.jobs.component.ICronjob import net.moonleay.rssbot.util.EmbedUtil import net.moonleay.rssbot.util.Logger import net.moonleay.rssbot.util.MessageUtil +import net.moonleay.rssbot.util.TimeUtil object FeedUpdater : ICronjob { override val jobName: String @@ -72,7 +74,13 @@ object FeedUpdater : ICronjob { if (rss == null) continue for (rssData in rss!!.items) { - if (rssData.guid == null || RSSRepository.exists(RSSData(rssData.guid!!, data.id))) + if (rssData.guid == null || RSSRepository.exists( + RSSData( + rssData.guid!!, + data.id + ) + ) || TimeUtil.getUnixTimeFromStamp(rssData.pubDate!!) < data.subscriptionTimestamp + ) continue val channel = Bot.bot.kordRef.getChannel(Snowflake(data.channelId))!!.asChannelOf() channel.createMessage { @@ -81,11 +89,14 @@ object FeedUpdater : ICronjob { data.feedColor, rssData.author ?: "Anonymous", rssData.title ?: "Untitled", - rssData.description ?: "No description", + if (rssData.description == null) "No description" else Remark(EmbedUtil.getHTMLtoMarkdownOptions()).convertFragment( + rssData.description!! + ), rssData.link ?: "https://moonleay.net/", rssData.image ?: "", if (rssData.link != null) EmbedUtil.getSiteLogo(rssData.link!!) else "https://static.moonleay.net/img/no-image.png", - data.feedName, + "Manual Feed", + rssData.pubDate ?: "unknown" ) ) } From 5d14fa9a35241a8266c6174f8e4d893ad014c9cc Mon Sep 17 00:00:00 2001 From: moonleay Date: Sun, 15 Oct 2023 15:46:26 +0200 Subject: [PATCH 09/10] feat: added SendFeedExtension, CleanUpExtension Signed-off-by: moonleay --- src/main/kotlin/net/moonleay/rssbot/Bot.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/net/moonleay/rssbot/Bot.kt b/src/main/kotlin/net/moonleay/rssbot/Bot.kt index ee2decd..5a83d6e 100644 --- a/src/main/kotlin/net/moonleay/rssbot/Bot.kt +++ b/src/main/kotlin/net/moonleay/rssbot/Bot.kt @@ -24,9 +24,7 @@ import dev.kord.core.event.gateway.ReadyEvent import dev.kord.core.on import net.moonleay.rssbot.data.CredentialManager import net.moonleay.rssbot.data.database.DB -import net.moonleay.rssbot.extensions.FeedExtension -import net.moonleay.rssbot.extensions.FeedsExtension -import net.moonleay.rssbot.extensions.StuffExtension +import net.moonleay.rssbot.extensions.* import net.moonleay.rssbot.jobs.FeedUpdater import net.moonleay.rssbot.jobs.component.JobManager import net.moonleay.rssbot.util.Logger @@ -77,6 +75,8 @@ object Bot { add(::FeedExtension) add(::StuffExtension) add(::FeedsExtension) + add(::SendFeedExtension) + add(::CleanUpExtension) } this.presence { From 8f58b5b6e62b2f23426994152fc710f629211724 Mon Sep 17 00:00:00 2001 From: moonleay Date: Sun, 15 Oct 2023 15:46:40 +0200 Subject: [PATCH 10/10] chore: bump version Signed-off-by: moonleay --- build.gradle.kts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index c675a62..86d6a5b 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -14,7 +14,7 @@ val ownerID = 372703841151614976L group = "net.moonleay.rssbot" version = System.getenv("CI_COMMIT_TAG")?.let { "$it-${System.getenv("CI_COMMIT_SHORT_SHA")}-prod" } ?: System.getenv("CI_COMMIT_SHORT_SHA")?.let { "$it-dev" } - ?: "0.2.1" + ?: "0.3.0" val kordver = "1.5.10-SNAPSHOT" val coroutinesver = "1.7.3" @@ -67,6 +67,9 @@ dependencies { //rss shadow("com.prof18.rssparser:rssparser:$rssparserver") + + // html to markdown + shadow("com.kotcrab.remark:remark:1.2.0") }