Kleinanzeigen + Telegram Bot working + Ignore,Remember

This commit is contained in:
Patrick Haßel 2024-06-06 08:38:36 +02:00
commit 025c1d6560
11 changed files with 713 additions and 0 deletions

42
.gitignore vendored Normal file
View File

@ -0,0 +1,42 @@
/offers.json
/.idea/
/telegram.token
target/
!.mvn/wrapper/maven-wrapper.jar
!**/src/main/**/target/
!**/src/test/**/target/
### IntelliJ IDEA ###
.idea/modules.xml
.idea/jarRepositories.xml
.idea/compiler.xml
.idea/libraries/
*.iws
*.iml
*.ipr
### Eclipse ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
build/
!**/src/main/**/build/
!**/src/test/**/build/
### VS Code ###
.vscode/
### Mac OS ###
.DS_Store

45
pom.xml Normal file
View File

@ -0,0 +1,45 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>de.ph87</groupId>
<artifactId>Kleinanzeigen</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.32</version>
</dependency>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.16.1</version>
</dependency>
<dependency>
<groupId>org.telegram</groupId>
<artifactId>telegrambots</artifactId>
<version>6.9.7.1</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>2.0.12</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
<version>2.15.4</version>
</dependency>
</dependencies>
</project>

View File

@ -0,0 +1,198 @@
package de.ph87.kleinanzeigen.api;
import com.fasterxml.jackson.core.JsonProcessingException;
import lombok.extern.slf4j.Slf4j;
import org.telegram.telegrambots.bots.TelegramLongPollingBot;
import org.telegram.telegrambots.meta.TelegramBotsApi;
import org.telegram.telegrambots.meta.api.methods.send.SendMessage;
import org.telegram.telegrambots.meta.api.methods.send.SendPhoto;
import org.telegram.telegrambots.meta.api.methods.updatingmessages.DeleteMessages;
import org.telegram.telegrambots.meta.api.methods.updatingmessages.EditMessageCaption;
import org.telegram.telegrambots.meta.api.methods.updatingmessages.EditMessageText;
import org.telegram.telegrambots.meta.api.objects.*;
import org.telegram.telegrambots.meta.api.objects.replykeyboard.InlineKeyboardMarkup;
import org.telegram.telegrambots.meta.api.objects.replykeyboard.buttons.InlineKeyboardButton;
import org.telegram.telegrambots.meta.exceptions.TelegramApiException;
import org.telegram.telegrambots.updatesreceivers.DefaultBotSession;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import static de.ph87.kleinanzeigen.api.JSON.objectMapper;
@Slf4j
public class Bot extends TelegramLongPollingBot {
private static final long CHAT_ID = 101138682L;
private final DefaultBotSession session;
private final Consumer<MaybeInaccessibleMessage> ignore;
private final BiFunction<MaybeInaccessibleMessage, Boolean, Optional<Offer>> remember;
public Bot(final Consumer<MaybeInaccessibleMessage> ignore, final BiFunction<MaybeInaccessibleMessage, Boolean, Optional<Offer>> remember) throws IOException, TelegramApiException {
super(readToken());
this.ignore = ignore;
this.remember = remember;
log.info("Starting telegram bot...");
final TelegramBotsApi api = new TelegramBotsApi(DefaultBotSession.class);
session = (DefaultBotSession) api.registerBot(this);
log.info("Telegram bot registered.");
}
@Override
public String getBotUsername() {
return "BotKleinanzeigenBot";
}
@Override
public void onUpdateReceived(final Update update) {
if (update.hasMessage() && update.getMessage().hasText()) {
log.info("#{} \"{}\": {}", update.getMessage().getChat().getId(), update.getMessage().getChat().getUserName(), update.getMessage().getText());
} else if (update.hasCallbackQuery()) {
handleCallbackQuery(update.getCallbackQuery());
}
}
private void handleCallbackQuery(final CallbackQuery query) {
final MaybeInaccessibleMessage message = query.getMessage();
if (message.getChatId() != CHAT_ID) {
return;
}
try {
final InlineDto dto = objectMapper.readValue(query.getData(), InlineDto.class);
switch (dto.getCommand()) {
case IGNORE -> ignore(message);
case REMEMBER -> remember(message, dto);
case UNREMEMBER -> unremember(message, dto);
}
} catch (JsonProcessingException e) {
log.error("Failed to read InlineDto.", e);
}
}
private void ignore(final MaybeInaccessibleMessage message) {
ignore.accept(message);
remove(message);
}
private void remember(final MaybeInaccessibleMessage message, final InlineDto dto) {
remember.apply(message, true).ifPresent(offer -> editMessage(message, offer, dto));
}
private void unremember(final MaybeInaccessibleMessage message, final InlineDto dto) {
remember.apply(message, false).ifPresent(offer -> editMessage(message, offer, dto));
}
private void editMessage(final MaybeInaccessibleMessage message, final Offer offer, final InlineDto dto) {
try {
switch (dto.getType()) {
case TEXT -> {
final EditMessageText edit = new EditMessageText(message.getChatId() + "", message.getMessageId(), null, getText(offer), null, null, getKeyboard(offer), null, null);
edit.setParseMode("Markdown");
execute(edit);
}
case PHOTO -> {
final EditMessageCaption edit = new EditMessageCaption(message.getChatId() + "", message.getMessageId(), null, getText(offer), getKeyboard(offer), null, null);
edit.setParseMode("Markdown");
execute(edit);
}
}
} catch (TelegramApiException | JsonProcessingException e) {
log.error("Failed to edit Message to #{}.", CHAT_ID, e);
}
}
private static String readToken() throws IOException {
try (final FileInputStream stream = new FileInputStream("./telegram.token")) {
return new String(stream.readAllBytes(), StandardCharsets.UTF_8).trim();
}
}
public void send(final Offer offer) {
try {
final Message message;
if (offer.getImage().isEmpty()) {
final SendMessage send = new SendMessage(CHAT_ID + "", getText(offer));
send.setParseMode("Markdown");
send.setReplyMarkup(getKeyboard(offer));
message = execute(send);
} else {
final SendPhoto send = new SendPhoto(CHAT_ID + "", new InputFile(offer.getImage()));
send.setCaption(getText(offer));
send.setParseMode("Markdown");
send.setReplyMarkup(getKeyboard(offer));
message = execute(send);
}
offer.setTelegramMessageId(message.getMessageId());
} catch (TelegramApiException | JsonProcessingException e) {
log.error("Failed to send Message to #{}.", CHAT_ID, e);
}
}
private String getText(final Offer offer) {
return "[%s](%s)\nVor %s\n%s\n%s".formatted(
escape(offer.getTitle()),
offer.getHref(),
offer.ageString(),
offer.calculateLocationString(),
offer.getDescription()
);
}
private String escape(final String text) {
// TODO make smarter (real escaping)
return text.replaceAll("\\[", "(").replaceAll("]", ")");
}
private InlineKeyboardMarkup getKeyboard(final Offer offer) throws JsonProcessingException {
final InlineKeyboardMarkup markup = new InlineKeyboardMarkup();
final ArrayList<List<InlineKeyboardButton>> keyboard = new ArrayList<>();
final ArrayList<InlineKeyboardButton> row = new ArrayList<>();
if (offer.isRemember()) {
addButton(row, "Nicht mehr merken", InlineCommand.UNREMEMBER, offer);
} else {
addButton(row, "Ignorieren", InlineCommand.IGNORE, offer);
addButton(row, "Merken", InlineCommand.REMEMBER, offer);
}
keyboard.add(row);
markup.setKeyboard(keyboard);
return markup;
}
private void addButton(final ArrayList<InlineKeyboardButton> row, final String caption, final InlineCommand command, final Offer offer) throws JsonProcessingException {
final InlineDto dto = new InlineDto(command, InlineType.of(offer));
final String data = objectMapper.writeValueAsString(dto);
row.add(new InlineKeyboardButton(caption, null, data, null, null, null, null, null, null));
}
public void remove(final List<Offer> offers) {
_remove(offers.stream().map(Offer::getTelegramMessageId).filter(Objects::nonNull).toList());
}
private void remove(final MaybeInaccessibleMessage... messages) {
_remove(Arrays.stream(messages).map(MaybeInaccessibleMessage::getMessageId).toList());
}
private void _remove(final List<Integer> messageIds) {
if (messageIds.isEmpty()) {
return;
}
try {
execute(new DeleteMessages(CHAT_ID + "", messageIds));
} catch (TelegramApiException e) {
log.error("Failed to remove Message to #{}.", CHAT_ID, e);
}
}
public void stop() {
session.stop();
}
}

View File

@ -0,0 +1,5 @@
package de.ph87.kleinanzeigen.api;
public enum InlineCommand {
IGNORE, REMEMBER, UNREMEMBER
}

View File

@ -0,0 +1,19 @@
package de.ph87.kleinanzeigen.api;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
public class InlineDto {
private InlineCommand command;
private InlineType type;
public InlineDto(final InlineCommand command, final InlineType type) {
this.command = command;
this.type = type;
}
}

View File

@ -0,0 +1,13 @@
package de.ph87.kleinanzeigen.api;
public enum InlineType {
TEXT, PHOTO;
public static InlineType of(final Offer offer) {
if (offer.getImage() == null) {
return TEXT;
}
return PHOTO;
}
}

View File

@ -0,0 +1,9 @@
package de.ph87.kleinanzeigen.api;
import com.fasterxml.jackson.databind.ObjectMapper;
public class JSON {
public static final ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules();
}

View File

@ -0,0 +1,198 @@
package de.ph87.kleinanzeigen.api;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.telegram.telegrambots.meta.api.objects.MaybeInaccessibleMessage;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.time.LocalDate;
import java.time.LocalTime;
import java.time.ZonedDateTime;
import java.util.*;
import java.util.function.Consumer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static de.ph87.kleinanzeigen.api.JSON.objectMapper;
@Slf4j
public class Kleinanzeigen {
private static final int KEEP_LAST_OFFERS_COUNT = 50;
private static final File FILE = new File("./offers.json");
private static final URI VERSCHENKEN_EPPELBORN_30KM = URI.create("https://www.kleinanzeigen.de/s-zu-verschenken/66571/c192l339r30");
private final List<Offer> offers;
private final Consumer<Offer> remove;
public Kleinanzeigen(final Consumer<Offer> remove) {
this.remove = remove;
offers = load();
}
private List<Offer> load() {
try {
final List<Offer> offers = objectMapper.readerForListOf(Offer.class).readValue(FILE);
log.info("Loaded {} offers from file: {}", offers.size(), FILE);
return offers;
} catch (IOException e) {
log.warn("Failed to load Offers from file={}: {}", FILE, e.toString());
return new ArrayList<>();
}
}
private void save() {
try {
final List<Offer> removed;
synchronized (offers) {
removed = _cleanUp();
objectMapper.writerWithDefaultPrettyPrinter().writeValue(FILE, offers);
log.info("Wrote {} offers to file: {}", offers.size(), FILE);
}
removed.forEach(remove);
} catch (IOException e) {
log.warn("Failed to write Offers to file={}: {}", FILE, e.toString());
}
}
private List<Offer> _cleanUp() {
if (offers.stream().anyMatch(Offer::_deleted_)) {
throw new RuntimeException();
}
offers.sort(Comparator.comparing(Offer::getDate));
final List<Offer> deleted = new ArrayList<>();
final List<Offer> notToRemember = new ArrayList<>(offers.stream().filter(offer -> !offer.isRemember()).toList());
while (!notToRemember.isEmpty() && notToRemember.size() > Kleinanzeigen.KEEP_LAST_OFFERS_COUNT) {
final Offer offer = notToRemember.removeFirst();
offers.remove(offer);
offer.markDeleted();
deleted.add(offer);
}
return deleted;
}
public void fetch() {
try {
final Document document = Jsoup.parse(VERSCHENKEN_EPPELBORN_30KM.toURL(), 3000);
for (Element article : document.select("li.ad-listitem:not(.is-topad) article.aditem")) {
try {
final Offer offer = parse(article);
merge(offer);
} catch (OfferParseException e) {
log.error("Failed to parse Offer:", e);
}
}
save();
} catch (IOException e) {
log.error("Failed to fetch Kleinanzeigen: {}", e.toString());
}
}
private Offer parse(final Element article) throws OfferParseException {
try {
final String id = article.attr("data-adid");
final String title = article.select(".text-module-begin").text();
final String description = article.select(".aditem-main--middle--description").text();
final ZonedDateTime date = parseDate(article.select(".aditem-main--top--right").text());
final String articleURL = VERSCHENKEN_EPPELBORN_30KM.resolve(article.select(".aditem-image a").attr("href")).toString();
final String imageURL = article.select(".aditem-image img").attr("src");
final String zipcode;
final String location;
final Integer distance;
final String locationString = article.select(".aditem-main--top--left").text();
final Matcher locationMatcher = Pattern.compile("^(?<zipcode>\\d+) (?<location>.+) \\((:?ca.)?\\s*(?<distance>\\d+)\\s*km\\s*\\)$").matcher(locationString);
if (!locationMatcher.find()) {
zipcode = "";
location = locationString;
distance = null;
} else {
zipcode = locationMatcher.group("zipcode");
location = locationMatcher.group("location");
distance = Integer.parseInt(locationMatcher.group("distance"));
}
return new Offer(id, date, title, zipcode, location, distance, description, articleURL, imageURL);
} catch (NumberFormatException e) {
throw new OfferParseException(article, e);
}
}
private void merge(final Offer offer) {
synchronized (offer) {
offers.stream()
.filter(existing -> existing.getId().equals(offer.getId()))
.peek(existing -> {
existing.merge(offer);
log.info("Updated: {}", existing);
})
.findFirst()
.orElseGet(() -> {
log.info("Created: {}", offer);
offers.add(offer);
return offer;
});
}
}
private ZonedDateTime parseDate(final String text) {
final Matcher dayNameMatcher = Pattern.compile("(?<day>Gestern|Heute), (?<hour>\\d+):(?<minute>\\d+)").matcher(text);
if (dayNameMatcher.find()) {
final long minusDays = dayNameMatcher.group("day").equals("Gestern") ? 1 : 0;
return ZonedDateTime.now().minusDays(minusDays).withHour(Integer.parseInt(dayNameMatcher.group("hour"))).withMinute(Integer.parseInt(dayNameMatcher.group("minute"))).withSecond(0).withNano(0);
}
final Matcher localDateMatcher = Pattern.compile("(?<day>\\d+).(?<month>\\d+).(?<year>\\d+)").matcher(text);
if (localDateMatcher.find()) {
return ZonedDateTime.of(
LocalDate.of(
Integer.parseInt(localDateMatcher.group("day")),
Integer.parseInt(localDateMatcher.group("month")),
Integer.parseInt(localDateMatcher.group("year"))
),
LocalTime.MIDNIGHT,
TimeZone.getDefault().toZoneId()
);
}
throw new NumberFormatException("Failed to parse date: " + text);
}
public List<Offer> findAll() {
synchronized (offers) {
return new ArrayList<>(offers);
}
}
public void ignore(final MaybeInaccessibleMessage message) {
synchronized (offers) {
findByTelegramMessageId(message).ifPresent(offer -> {
offer.ignore();
save();
});
}
}
public Optional<Offer> remember(final MaybeInaccessibleMessage message, final boolean remember) {
synchronized (offers) {
final Optional<Offer> optional = findByTelegramMessageId(message);
optional.ifPresent(offer -> {
offer.setRemember(remember);
save();
});
return optional;
}
}
private Optional<Offer> findByTelegramMessageId(final MaybeInaccessibleMessage message) {
return offers.stream().filter(offer -> Objects.equals(offer.getTelegramMessageId(), message.getMessageId())).findFirst();
}
}

View File

@ -0,0 +1,43 @@
package de.ph87.kleinanzeigen.api;
import lombok.extern.slf4j.Slf4j;
import org.telegram.telegrambots.meta.exceptions.TelegramApiException;
import java.io.IOException;
import java.util.List;
@Slf4j
@SuppressWarnings({"InfiniteLoopStatement", "SameParameterValue", "SynchronizationOnLocalVariableOrMethodParameter"})
public class Main {
private static Bot bot;
private static final Kleinanzeigen kleinanzeigen = new Kleinanzeigen(offer -> bot.remove(List.of(offer)));
public static void main(String[] args) throws IOException, TelegramApiException {
bot = new Bot(kleinanzeigen::ignore, kleinanzeigen::remember);
try {
while (true) {
handle(bot);
waitSeconds(60);
}
} catch (InterruptedException e) {
log.warn(e.toString());
} finally {
bot.stop();
}
}
private static void handle(final Bot bot) {
kleinanzeigen.fetch();
kleinanzeigen.findAll().stream().filter(offer -> offer.getTelegramMessageId() == null).forEach(bot::send);
}
private static void waitSeconds(final long seconds) throws InterruptedException {
final Object lock = new Object();
synchronized (lock) {
lock.wait(seconds * 1000);
}
}
}

View File

@ -0,0 +1,130 @@
package de.ph87.kleinanzeigen.api;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import java.time.Duration;
import java.time.ZonedDateTime;
@Getter
@NoArgsConstructor
public class Offer {
private String id;
@JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX")
private ZonedDateTime first = ZonedDateTime.now();
@JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX")
private ZonedDateTime last = first;
@JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX")
private ZonedDateTime date;
private String title;
private String zipcode;
private String location;
private Integer distance;
private String description;
private String href;
private String image;
@Setter
private Integer telegramMessageId = null;
private boolean ignore = false;
@Setter
private boolean remember = false;
@JsonIgnore
private boolean _deleted_ = false;
public Offer(final String id, final ZonedDateTime date, final String title, final String zipcode, final String location, final Integer distance, final String description, final String href, final String image) {
this.id = id;
this.date = date;
this.title = title;
this.zipcode = zipcode;
this.location = location;
this.distance = distance;
this.description = description;
this.href = href;
this.image = image;
}
@Override
public String toString() {
return "Offer(%s%s, %s, %s)".formatted(_deleted_ ? "[DELETED], " : "", title, calculateLocationString(), href);
}
public void verifyNotDeleted() {
if (_deleted_) {
throw new RuntimeException();
}
}
public String ageString() {
final Duration age = Duration.between(date, ZonedDateTime.now());
if (age.toDays() > 0) {
return "%d Tag%s".formatted(age.toDays(), age.toDays() == 1 ? "" : "en");
} else if (age.toHours() > 0) {
return "%d Stunde%s".formatted(age.toHours(), age.toHours() == 1 ? "" : "n");
} else {
return "%d Minute%s".formatted(age.toMinutes(), age.toMinutes() == 1 ? "" : "n");
}
}
public void merge(final Offer other) {
verifyNotDeleted();
other.verifyNotDeleted();
if (!id.equals(other.id)) {
throw new RuntimeException();
}
this.date = other.date;
this.title = other.title;
this.zipcode = other.zipcode;
this.location = other.location;
this.distance = other.distance;
this.description = other.description;
this.href = other.href;
this.image = other.image;
this.first = other.first.isBefore(this.first) ? other.first : this.first;
this.last = other.last.isAfter(this.last) ? other.last : this.last;
other.markDeleted();
}
public void markDeleted() {
_deleted_ = true;
}
public boolean _deleted_() {
return _deleted_;
}
public String calculateLocationString() {
String result = zipcode;
if (!result.isEmpty()) {
result += " ";
}
result += location;
if (distance != null) {
result += " (" + distance + " km)";
}
return result;
}
public void ignore() {
ignore = true;
remember = false;
}
}

View File

@ -0,0 +1,11 @@
package de.ph87.kleinanzeigen.api;
import org.jsoup.nodes.Element;
public class OfferParseException extends Exception {
public OfferParseException(final Element element, final RuntimeException cause) {
super(element.outerHtml(), cause);
}
}