diff --git a/build.gradle.kts b/build.gradle.kts
index c675a62..86d6a5b 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -14,7 +14,7 @@ val ownerID = 372703841151614976L
group = "net.moonleay.rssbot"
version = System.getenv("CI_COMMIT_TAG")?.let { "$it-${System.getenv("CI_COMMIT_SHORT_SHA")}-prod" }
?: System.getenv("CI_COMMIT_SHORT_SHA")?.let { "$it-dev" }
- ?: "0.2.1"
+ ?: "0.3.0"
val kordver = "1.5.10-SNAPSHOT"
val coroutinesver = "1.7.3"
@@ -67,6 +67,9 @@ dependencies {
//rss
shadow("com.prof18.rssparser:rssparser:$rssparserver")
+
+ // html to markdown
+ shadow("com.kotcrab.remark:remark:1.2.0")
}
diff --git a/src/main/kotlin/net/moonleay/rssbot/Bot.kt b/src/main/kotlin/net/moonleay/rssbot/Bot.kt
index ee2decd..5a83d6e 100644
--- a/src/main/kotlin/net/moonleay/rssbot/Bot.kt
+++ b/src/main/kotlin/net/moonleay/rssbot/Bot.kt
@@ -24,9 +24,7 @@ import dev.kord.core.event.gateway.ReadyEvent
import dev.kord.core.on
import net.moonleay.rssbot.data.CredentialManager
import net.moonleay.rssbot.data.database.DB
-import net.moonleay.rssbot.extensions.FeedExtension
-import net.moonleay.rssbot.extensions.FeedsExtension
-import net.moonleay.rssbot.extensions.StuffExtension
+import net.moonleay.rssbot.extensions.*
import net.moonleay.rssbot.jobs.FeedUpdater
import net.moonleay.rssbot.jobs.component.JobManager
import net.moonleay.rssbot.util.Logger
@@ -77,6 +75,8 @@ object Bot {
add(::FeedExtension)
add(::StuffExtension)
add(::FeedsExtension)
+ add(::SendFeedExtension)
+ add(::CleanUpExtension)
}
this.presence {
diff --git a/src/main/kotlin/net/moonleay/rssbot/data/database/entry/SubscriptionData.kt b/src/main/kotlin/net/moonleay/rssbot/data/database/entry/SubscriptionData.kt
index 71f47b5..3896b37 100644
--- a/src/main/kotlin/net/moonleay/rssbot/data/database/entry/SubscriptionData.kt
+++ b/src/main/kotlin/net/moonleay/rssbot/data/database/entry/SubscriptionData.kt
@@ -27,4 +27,5 @@ data class SubscriptionData(
val feedColor: FeedColor,
val feedName: String,
val feedUrl: String,
+ val subscriptionTimestamp: Long,
)
diff --git a/src/main/kotlin/net/moonleay/rssbot/data/database/repository/SubscriptionRepository.kt b/src/main/kotlin/net/moonleay/rssbot/data/database/repository/SubscriptionRepository.kt
index f80d96b..95cc5fb 100644
--- a/src/main/kotlin/net/moonleay/rssbot/data/database/repository/SubscriptionRepository.kt
+++ b/src/main/kotlin/net/moonleay/rssbot/data/database/repository/SubscriptionRepository.kt
@@ -38,7 +38,8 @@ object SubscriptionRepository {
it[SubscriptionsTable.channelId],
EmbedUtil.getColorFromString(it[SubscriptionsTable.subscriptionColor]),
it[SubscriptionsTable.subscriptionName],
- it[SubscriptionsTable.feedUrl]
+ it[SubscriptionsTable.feedUrl],
+ it[SubscriptionsTable.subscriptionTimestamp]
)
)
}
@@ -59,7 +60,8 @@ object SubscriptionRepository {
it[SubscriptionsTable.channelId],
EmbedUtil.getColorFromString(it[SubscriptionsTable.subscriptionColor]),
it[SubscriptionsTable.subscriptionName],
- it[SubscriptionsTable.feedUrl]
+ it[SubscriptionsTable.feedUrl],
+ it[SubscriptionsTable.subscriptionTimestamp]
)
)
}
@@ -80,7 +82,8 @@ object SubscriptionRepository {
it[SubscriptionsTable.channelId],
EmbedUtil.getColorFromString(it[SubscriptionsTable.subscriptionColor]),
it[SubscriptionsTable.subscriptionName],
- it[SubscriptionsTable.feedUrl]
+ it[SubscriptionsTable.feedUrl],
+ it[SubscriptionsTable.subscriptionTimestamp]
)
}
}
@@ -115,6 +118,7 @@ object SubscriptionRepository {
it[subscriptionColor] = data.feedColor.readableName
it[subscriptionName] = data.feedName
it[feedUrl] = data.feedUrl
+ it[subscriptionTimestamp] = data.subscriptionTimestamp
}
}
diff --git a/src/main/kotlin/net/moonleay/rssbot/data/database/tables/SubscriptionsTable.kt b/src/main/kotlin/net/moonleay/rssbot/data/database/tables/SubscriptionsTable.kt
index de0ce39..ada2744 100644
--- a/src/main/kotlin/net/moonleay/rssbot/data/database/tables/SubscriptionsTable.kt
+++ b/src/main/kotlin/net/moonleay/rssbot/data/database/tables/SubscriptionsTable.kt
@@ -28,4 +28,5 @@ object SubscriptionsTable : Table(name = "subscriptions") {
val subscriptionColor = text("subscription_color")
val subscriptionName = text("subscription_name")
val feedUrl = text("feed_url")
+ val subscriptionTimestamp = long("subscription_timestamp")
}
diff --git a/src/main/kotlin/net/moonleay/rssbot/extensions/CleanUpExtension.kt b/src/main/kotlin/net/moonleay/rssbot/extensions/CleanUpExtension.kt
new file mode 100644
index 0000000..f2abff6
--- /dev/null
+++ b/src/main/kotlin/net/moonleay/rssbot/extensions/CleanUpExtension.kt
@@ -0,0 +1,39 @@
+package net.moonleay.rssbot.extensions
+
+import com.kotlindiscord.kord.extensions.extensions.Extension
+import com.kotlindiscord.kord.extensions.extensions.ephemeralSlashCommand
+import com.kotlindiscord.kord.extensions.types.respond
+import kotlinx.coroutines.flow.filter
+import net.moonleay.rssbot.Bot
+import net.moonleay.rssbot.data.database.repository.SubscriptionRepository
+import net.moonleay.rssbot.util.EmbedColor
+import net.moonleay.rssbot.util.MessageUtil
+
+class CleanUpExtension : Extension() {
+
+ override val name = "cleanup"
+ override val allowApplicationCommandInDMs: Boolean
+ get() = false
+
+
+ override suspend fun setup() {
+ ephemeralSlashCommand() {
+ name = "cleanup"
+ description = "Clean all messages from this bot in this channel"
+ this.action {
+ val user = this.user.asUser()
+ this.channel.messages.filter { it.author?.id == Bot.bot.kordRef.selfId }.collect { it.delete() }
+ this.respond {
+ this.embeds.add(
+ MessageUtil.getEmbed(
+ EmbedColor.INFO,
+ "Cleaned up",
+ "Cleaned up all messages from this bot in this channel",
+ user.username
+ )
+ )
+ }
+ }
+ }
+ }
+}
diff --git a/src/main/kotlin/net/moonleay/rssbot/extensions/FeedExtension.kt b/src/main/kotlin/net/moonleay/rssbot/extensions/FeedExtension.kt
index 06b40d1..dcfbe81 100644
--- a/src/main/kotlin/net/moonleay/rssbot/extensions/FeedExtension.kt
+++ b/src/main/kotlin/net/moonleay/rssbot/extensions/FeedExtension.kt
@@ -106,7 +106,8 @@ class FeedExtension : Extension() {
this.channel.id.value.toLong(),
feedColor,
feedName,
- feedUrl
+ feedUrl,
+ System.currentTimeMillis()
)
)
Logger.out("added feed as id ${id.resultedValues!!.first()[SubscriptionsTable.id]}")
diff --git a/src/main/kotlin/net/moonleay/rssbot/extensions/SendFeedExtension.kt b/src/main/kotlin/net/moonleay/rssbot/extensions/SendFeedExtension.kt
new file mode 100644
index 0000000..11cf8d9
--- /dev/null
+++ b/src/main/kotlin/net/moonleay/rssbot/extensions/SendFeedExtension.kt
@@ -0,0 +1,113 @@
+/*
+ * RSSBot
+ * Copyright (C) 2023 moonleay
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see .
+ */
+
+package net.moonleay.rssbot.extensions
+
+import com.kotlindiscord.kord.extensions.commands.Arguments
+import com.kotlindiscord.kord.extensions.commands.converters.impl.string
+import com.kotlindiscord.kord.extensions.extensions.Extension
+import com.kotlindiscord.kord.extensions.extensions.ephemeralSlashCommand
+import com.kotlindiscord.kord.extensions.types.respond
+import com.overzealous.remark.Options
+import com.overzealous.remark.Remark
+import com.prof18.rssparser.RssParser
+import com.prof18.rssparser.model.RssChannel
+import com.prof18.rssparser.model.RssItem
+import dev.kord.rest.builder.message.EmbedBuilder.Limits.title
+import net.moonleay.rssbot.data.database.repository.RSSRepository
+import net.moonleay.rssbot.data.database.repository.SubscriptionRepository
+import net.moonleay.rssbot.extensions.components.FeedColor
+import net.moonleay.rssbot.util.EmbedColor
+import net.moonleay.rssbot.util.EmbedUtil
+import net.moonleay.rssbot.util.MessageUtil
+
+class SendFeedExtension : Extension() {
+
+ override val name = "sendfeed"
+ override val allowApplicationCommandInDMs: Boolean
+ get() = false
+
+
+ override suspend fun setup() {
+ ephemeralSlashCommand(::StuffArguments) {
+ name = "sendfeed"
+ description = "Send a feed to this channel"
+ this.action {
+ val feedUrl = arguments.feedUrl
+ val user = this.user.asUser()
+ if (feedUrl == null) {
+ this.respond {
+ this.embeds.add(
+ MessageUtil.getEmbed(
+ EmbedColor.ERROR,
+ "Missing Arguments",
+ "You are missing one or more arguments",
+ user.username
+ )
+ )
+ }
+ return@action
+ }
+ val parser = RssParser()
+ var rss: RssChannel? = null
+ runCatching { // this sucks
+ rss = parser.getRssChannel(feedUrl)
+ }.onFailure {
+ this.respond {
+ this.embeds.add(
+ MessageUtil.getEmbed(
+ EmbedColor.ERROR,
+ "Invalid Feed",
+ "The feed you provided is invalid",
+ user.username
+ )
+ )
+ }
+ return@action
+ }
+ if (rss == null)
+ return@action
+ rss!!.items.forEach {
+ this.respond {
+ this.embeds.add(
+ MessageUtil.getRSSEmbed(
+ FeedColor.GREEN,
+ it.author ?: "Anonymous",
+ it.title ?: "Untitled",
+ if (it.description == null) "No description" else Remark(EmbedUtil.getHTMLtoMarkdownOptions()).convertFragment(it.description!!),
+ it.link ?: "https://moonleay.net/",
+ it.image ?: "",
+ if (it.link != null) EmbedUtil.getSiteLogo(it.link!!) else "https://static.moonleay.net/img/no-image.png",
+ "Manual Feed",
+ it.pubDate?: "unknown"
+ )
+ )
+ }
+ }
+
+ }
+ }
+ }
+
+ inner class StuffArguments : Arguments() {
+ val feedUrl by string {
+ this.name = "url"
+ this.description = "The url of the feed to send"
+ }
+ }
+}
diff --git a/src/main/kotlin/net/moonleay/rssbot/jobs/FeedUpdater.kt b/src/main/kotlin/net/moonleay/rssbot/jobs/FeedUpdater.kt
index ff712b6..810175c 100644
--- a/src/main/kotlin/net/moonleay/rssbot/jobs/FeedUpdater.kt
+++ b/src/main/kotlin/net/moonleay/rssbot/jobs/FeedUpdater.kt
@@ -18,6 +18,7 @@
package net.moonleay.rssbot.jobs
+import com.overzealous.remark.Remark
import com.prof18.rssparser.RssParser
import com.prof18.rssparser.model.RssChannel
import dev.inmo.krontab.KronScheduler
@@ -35,6 +36,7 @@ import net.moonleay.rssbot.jobs.component.ICronjob
import net.moonleay.rssbot.util.EmbedUtil
import net.moonleay.rssbot.util.Logger
import net.moonleay.rssbot.util.MessageUtil
+import net.moonleay.rssbot.util.TimeUtil
object FeedUpdater : ICronjob {
override val jobName: String
@@ -72,7 +74,13 @@ object FeedUpdater : ICronjob {
if (rss == null)
continue
for (rssData in rss!!.items) {
- if (rssData.guid == null || RSSRepository.exists(RSSData(rssData.guid!!, data.id)))
+ if (rssData.guid == null || RSSRepository.exists(
+ RSSData(
+ rssData.guid!!,
+ data.id
+ )
+ ) || TimeUtil.getUnixTimeFromStamp(rssData.pubDate!!) < data.subscriptionTimestamp
+ )
continue
val channel = Bot.bot.kordRef.getChannel(Snowflake(data.channelId))!!.asChannelOf()
channel.createMessage {
@@ -81,11 +89,14 @@ object FeedUpdater : ICronjob {
data.feedColor,
rssData.author ?: "Anonymous",
rssData.title ?: "Untitled",
- rssData.description ?: "No description",
+ if (rssData.description == null) "No description" else Remark(EmbedUtil.getHTMLtoMarkdownOptions()).convertFragment(
+ rssData.description!!
+ ),
rssData.link ?: "https://moonleay.net/",
rssData.image ?: "",
if (rssData.link != null) EmbedUtil.getSiteLogo(rssData.link!!) else "https://static.moonleay.net/img/no-image.png",
- data.feedName,
+ "Manual Feed",
+ rssData.pubDate ?: "unknown"
)
)
}
diff --git a/src/main/kotlin/net/moonleay/rssbot/util/EmbedUtil.kt b/src/main/kotlin/net/moonleay/rssbot/util/EmbedUtil.kt
index 6ce00ca..634d3ce 100644
--- a/src/main/kotlin/net/moonleay/rssbot/util/EmbedUtil.kt
+++ b/src/main/kotlin/net/moonleay/rssbot/util/EmbedUtil.kt
@@ -18,6 +18,8 @@
package net.moonleay.rssbot.util
+import com.overzealous.remark.IgnoredHtmlElement
+import com.overzealous.remark.Options
import dev.kord.common.entity.ButtonStyle
import dev.kord.core.entity.Embed
import dev.kord.rest.builder.component.ActionRowBuilder
@@ -141,4 +143,11 @@ object EmbedUtil {
return "https://www.google.com/s2/favicons?domain=${str.split("/")[2]}"
}
+ fun getHTMLtoMarkdownOptions(): Options {
+ val o = Options()
+ o.ignoredHtmlElements.add(IgnoredHtmlElement("img"))
+ o.inlineLinks = true
+ return o
+ }
+
}
diff --git a/src/main/kotlin/net/moonleay/rssbot/util/MessageUtil.kt b/src/main/kotlin/net/moonleay/rssbot/util/MessageUtil.kt
index d6d8ace..e01058b 100644
--- a/src/main/kotlin/net/moonleay/rssbot/util/MessageUtil.kt
+++ b/src/main/kotlin/net/moonleay/rssbot/util/MessageUtil.kt
@@ -174,11 +174,12 @@ object MessageUtil {
imageUrl: String,
logo: String,
feedName: String,
+ published: String,
): EmbedBuilder {
val ebb = EmbedBuilder()
val now: LocalDateTime = LocalDateTime.now()
- if (title.length > 250)
- ebb.title = title.substring(0, 250) + "[...]"
+ if (title.length > 25)
+ ebb.title = title.substring(0, 25) + "..."
else
ebb.title = title
ebb.author {
@@ -188,7 +189,7 @@ object MessageUtil {
ebb.description = "${if (description != "No description") description else ""}\n[[open full article]]($url)"
ebb.color = color.color.color
ebb.footer {
- this.text = "> rssbot, ($feedName) / sent at ${dtf.format(now)}"
+ this.text = "> rssbot, ($feedName) \n/ pub $published \n/ rel ${dtf.format(now)}"
this.icon = "https://static.moonleay.net/img/rss.png"
}
ebb.thumbnail {
diff --git a/src/main/kotlin/net/moonleay/rssbot/util/TimeUtil.kt b/src/main/kotlin/net/moonleay/rssbot/util/TimeUtil.kt
index 89ff4b8..416d3a6 100644
--- a/src/main/kotlin/net/moonleay/rssbot/util/TimeUtil.kt
+++ b/src/main/kotlin/net/moonleay/rssbot/util/TimeUtil.kt
@@ -18,47 +18,13 @@
package net.moonleay.rssbot.util
-import kotlinx.datetime.DayOfWeek
import java.time.ZonedDateTime
-import java.util.concurrent.TimeUnit
+import java.time.format.DateTimeFormatter
object TimeUtil {
-
- fun getTimeFormatedShortend(time2: Long, showS: Boolean): String {
- var time = time2
- val days: Long = TimeUnit.MILLISECONDS
- .toDays(time)
- time -= TimeUnit.DAYS.toMillis(days)
- val hours: Long = TimeUnit.MILLISECONDS
- .toHours(time)
- time -= TimeUnit.HOURS.toMillis(hours)
- val minutes: Long = TimeUnit.MILLISECONDS
- .toMinutes(time)
- time -= TimeUnit.MINUTES.toMillis(minutes)
- val seconds: Long = TimeUnit.MILLISECONDS
- .toSeconds(time)
- var s = ""
- if (days >= 1) {
- s += days.toString() + "d "
- }
- if (hours >= 1) {
- s += hours.toString() + "h "
- }
- if (minutes >= 1) {
- s += minutes.toString() + "m "
- }
- if (seconds >= 1 && hours < 1 && showS) {
- s += seconds.toString() + "s"
- }
- if (s.isEmpty() || s.isBlank()) {
- s = "None"
- }
- return s
- }
-
- // Returns the day of the month of the monday of this week
- fun getMondayDayOfMonth(): Int {
- return ZonedDateTime.now().with(DayOfWeek.MONDAY).dayOfMonth
+ fun getUnixTimeFromStamp(input: String): Long { // Pattern: Sun, 15 Oct 2023 11:04:57 GMT
+ val formatter = DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss z")
+ return ZonedDateTime.parse(input, formatter).toEpochSecond() * 1000
}
}