commit 025c1d656052de6f39890c8755e878d4f6aeb01a Author: Patrick Haßel Date: Thu Jun 6 08:38:36 2024 +0200 Kleinanzeigen + Telegram Bot working + Ignore,Remember diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d87248d --- /dev/null +++ b/.gitignore @@ -0,0 +1,42 @@ +/offers.json +/.idea/ +/telegram.token + +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### IntelliJ IDEA ### +.idea/modules.xml +.idea/jarRepositories.xml +.idea/compiler.xml +.idea/libraries/ +*.iws +*.iml +*.ipr + +### Eclipse ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ + +### Mac OS ### +.DS_Store \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..8d75d22 --- /dev/null +++ b/pom.xml @@ -0,0 +1,45 @@ + + + 4.0.0 + + de.ph87 + Kleinanzeigen + 1.0-SNAPSHOT + + + 21 + 21 + UTF-8 + + + + + org.projectlombok + lombok + 1.18.32 + + + org.jsoup + jsoup + 1.16.1 + + + org.telegram + telegrambots + 6.9.7.1 + + + org.slf4j + slf4j-simple + 2.0.12 + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + 2.15.4 + + + + \ No newline at end of file diff --git a/src/main/java/de/ph87/kleinanzeigen/api/Bot.java b/src/main/java/de/ph87/kleinanzeigen/api/Bot.java new file mode 100644 index 0000000..fd319ea --- /dev/null +++ b/src/main/java/de/ph87/kleinanzeigen/api/Bot.java @@ -0,0 +1,198 @@ +package de.ph87.kleinanzeigen.api; + +import com.fasterxml.jackson.core.JsonProcessingException; +import lombok.extern.slf4j.Slf4j; +import org.telegram.telegrambots.bots.TelegramLongPollingBot; +import org.telegram.telegrambots.meta.TelegramBotsApi; +import org.telegram.telegrambots.meta.api.methods.send.SendMessage; +import org.telegram.telegrambots.meta.api.methods.send.SendPhoto; +import org.telegram.telegrambots.meta.api.methods.updatingmessages.DeleteMessages; +import org.telegram.telegrambots.meta.api.methods.updatingmessages.EditMessageCaption; +import org.telegram.telegrambots.meta.api.methods.updatingmessages.EditMessageText; +import org.telegram.telegrambots.meta.api.objects.*; +import org.telegram.telegrambots.meta.api.objects.replykeyboard.InlineKeyboardMarkup; +import org.telegram.telegrambots.meta.api.objects.replykeyboard.buttons.InlineKeyboardButton; +import org.telegram.telegrambots.meta.exceptions.TelegramApiException; +import org.telegram.telegrambots.updatesreceivers.DefaultBotSession; + +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.function.BiFunction; +import java.util.function.Consumer; + +import static de.ph87.kleinanzeigen.api.JSON.objectMapper; + +@Slf4j +public class Bot extends TelegramLongPollingBot { + + private static final long CHAT_ID = 101138682L; + + private final DefaultBotSession session; + + private final Consumer ignore; + + private final BiFunction> remember; + + public Bot(final Consumer ignore, final BiFunction> remember) throws IOException, TelegramApiException { + super(readToken()); + this.ignore = ignore; + this.remember = remember; + log.info("Starting telegram bot..."); + final TelegramBotsApi api = new TelegramBotsApi(DefaultBotSession.class); + session = (DefaultBotSession) api.registerBot(this); + log.info("Telegram bot registered."); + } + + @Override + public String getBotUsername() { + return "BotKleinanzeigenBot"; + } + + @Override + public void onUpdateReceived(final Update update) { + if (update.hasMessage() && update.getMessage().hasText()) { + log.info("#{} \"{}\": {}", update.getMessage().getChat().getId(), update.getMessage().getChat().getUserName(), update.getMessage().getText()); + } else if (update.hasCallbackQuery()) { + handleCallbackQuery(update.getCallbackQuery()); + } + } + + private void handleCallbackQuery(final CallbackQuery query) { + final MaybeInaccessibleMessage message = query.getMessage(); + if (message.getChatId() != CHAT_ID) { + return; + } + try { + final InlineDto dto = objectMapper.readValue(query.getData(), InlineDto.class); + switch (dto.getCommand()) { + case IGNORE -> ignore(message); + case REMEMBER -> remember(message, dto); + case UNREMEMBER -> unremember(message, dto); + } + } catch (JsonProcessingException e) { + log.error("Failed to read InlineDto.", e); + } + } + + private void ignore(final MaybeInaccessibleMessage message) { + ignore.accept(message); + remove(message); + } + + private void remember(final MaybeInaccessibleMessage message, final InlineDto dto) { + remember.apply(message, true).ifPresent(offer -> editMessage(message, offer, dto)); + } + + private void unremember(final MaybeInaccessibleMessage message, final InlineDto dto) { + remember.apply(message, false).ifPresent(offer -> editMessage(message, offer, dto)); + } + + private void editMessage(final MaybeInaccessibleMessage message, final Offer offer, final InlineDto dto) { + try { + switch (dto.getType()) { + case TEXT -> { + final EditMessageText edit = new EditMessageText(message.getChatId() + "", message.getMessageId(), null, getText(offer), null, null, getKeyboard(offer), null, null); + edit.setParseMode("Markdown"); + execute(edit); + } + case PHOTO -> { + final EditMessageCaption edit = new EditMessageCaption(message.getChatId() + "", message.getMessageId(), null, getText(offer), getKeyboard(offer), null, null); + edit.setParseMode("Markdown"); + execute(edit); + } + } + } catch (TelegramApiException | JsonProcessingException e) { + log.error("Failed to edit Message to #{}.", CHAT_ID, e); + } + } + + private static String readToken() throws IOException { + try (final FileInputStream stream = new FileInputStream("./telegram.token")) { + return new String(stream.readAllBytes(), StandardCharsets.UTF_8).trim(); + } + } + + public void send(final Offer offer) { + try { + final Message message; + if (offer.getImage().isEmpty()) { + final SendMessage send = new SendMessage(CHAT_ID + "", getText(offer)); + send.setParseMode("Markdown"); + send.setReplyMarkup(getKeyboard(offer)); + message = execute(send); + } else { + final SendPhoto send = new SendPhoto(CHAT_ID + "", new InputFile(offer.getImage())); + send.setCaption(getText(offer)); + send.setParseMode("Markdown"); + send.setReplyMarkup(getKeyboard(offer)); + message = execute(send); + } + + offer.setTelegramMessageId(message.getMessageId()); + } catch (TelegramApiException | JsonProcessingException e) { + log.error("Failed to send Message to #{}.", CHAT_ID, e); + } + } + + private String getText(final Offer offer) { + return "[%s](%s)\nVor %s\n%s\n%s".formatted( + escape(offer.getTitle()), + offer.getHref(), + offer.ageString(), + offer.calculateLocationString(), + offer.getDescription() + ); + } + + private String escape(final String text) { + // TODO make smarter (real escaping) + return text.replaceAll("\\[", "(").replaceAll("]", ")"); + } + + private InlineKeyboardMarkup getKeyboard(final Offer offer) throws JsonProcessingException { + final InlineKeyboardMarkup markup = new InlineKeyboardMarkup(); + final ArrayList> keyboard = new ArrayList<>(); + final ArrayList row = new ArrayList<>(); + if (offer.isRemember()) { + addButton(row, "Nicht mehr merken", InlineCommand.UNREMEMBER, offer); + } else { + addButton(row, "Ignorieren", InlineCommand.IGNORE, offer); + addButton(row, "Merken", InlineCommand.REMEMBER, offer); + } + keyboard.add(row); + markup.setKeyboard(keyboard); + return markup; + } + + private void addButton(final ArrayList row, final String caption, final InlineCommand command, final Offer offer) throws JsonProcessingException { + final InlineDto dto = new InlineDto(command, InlineType.of(offer)); + final String data = objectMapper.writeValueAsString(dto); + row.add(new InlineKeyboardButton(caption, null, data, null, null, null, null, null, null)); + } + + public void remove(final List offers) { + _remove(offers.stream().map(Offer::getTelegramMessageId).filter(Objects::nonNull).toList()); + } + + private void remove(final MaybeInaccessibleMessage... messages) { + _remove(Arrays.stream(messages).map(MaybeInaccessibleMessage::getMessageId).toList()); + } + + private void _remove(final List messageIds) { + if (messageIds.isEmpty()) { + return; + } + try { + execute(new DeleteMessages(CHAT_ID + "", messageIds)); + } catch (TelegramApiException e) { + log.error("Failed to remove Message to #{}.", CHAT_ID, e); + } + } + + public void stop() { + session.stop(); + } + +} diff --git a/src/main/java/de/ph87/kleinanzeigen/api/InlineCommand.java b/src/main/java/de/ph87/kleinanzeigen/api/InlineCommand.java new file mode 100644 index 0000000..8aa0c31 --- /dev/null +++ b/src/main/java/de/ph87/kleinanzeigen/api/InlineCommand.java @@ -0,0 +1,5 @@ +package de.ph87.kleinanzeigen.api; + +public enum InlineCommand { + IGNORE, REMEMBER, UNREMEMBER +} diff --git a/src/main/java/de/ph87/kleinanzeigen/api/InlineDto.java b/src/main/java/de/ph87/kleinanzeigen/api/InlineDto.java new file mode 100644 index 0000000..8950654 --- /dev/null +++ b/src/main/java/de/ph87/kleinanzeigen/api/InlineDto.java @@ -0,0 +1,19 @@ +package de.ph87.kleinanzeigen.api; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +public class InlineDto { + + private InlineCommand command; + + private InlineType type; + + public InlineDto(final InlineCommand command, final InlineType type) { + this.command = command; + this.type = type; + } + +} diff --git a/src/main/java/de/ph87/kleinanzeigen/api/InlineType.java b/src/main/java/de/ph87/kleinanzeigen/api/InlineType.java new file mode 100644 index 0000000..674f184 --- /dev/null +++ b/src/main/java/de/ph87/kleinanzeigen/api/InlineType.java @@ -0,0 +1,13 @@ +package de.ph87.kleinanzeigen.api; + +public enum InlineType { + TEXT, PHOTO; + + public static InlineType of(final Offer offer) { + if (offer.getImage() == null) { + return TEXT; + } + return PHOTO; + } + +} diff --git a/src/main/java/de/ph87/kleinanzeigen/api/JSON.java b/src/main/java/de/ph87/kleinanzeigen/api/JSON.java new file mode 100644 index 0000000..b98caac --- /dev/null +++ b/src/main/java/de/ph87/kleinanzeigen/api/JSON.java @@ -0,0 +1,9 @@ +package de.ph87.kleinanzeigen.api; + +import com.fasterxml.jackson.databind.ObjectMapper; + +public class JSON { + + public static final ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules(); + +} diff --git a/src/main/java/de/ph87/kleinanzeigen/api/Kleinanzeigen.java b/src/main/java/de/ph87/kleinanzeigen/api/Kleinanzeigen.java new file mode 100644 index 0000000..7cf3aa9 --- /dev/null +++ b/src/main/java/de/ph87/kleinanzeigen/api/Kleinanzeigen.java @@ -0,0 +1,198 @@ +package de.ph87.kleinanzeigen.api; + +import lombok.extern.slf4j.Slf4j; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.telegram.telegrambots.meta.api.objects.MaybeInaccessibleMessage; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZonedDateTime; +import java.util.*; +import java.util.function.Consumer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static de.ph87.kleinanzeigen.api.JSON.objectMapper; + +@Slf4j +public class Kleinanzeigen { + + private static final int KEEP_LAST_OFFERS_COUNT = 50; + + private static final File FILE = new File("./offers.json"); + + private static final URI VERSCHENKEN_EPPELBORN_30KM = URI.create("https://www.kleinanzeigen.de/s-zu-verschenken/66571/c192l339r30"); + + private final List offers; + + private final Consumer remove; + + public Kleinanzeigen(final Consumer remove) { + this.remove = remove; + offers = load(); + } + + private List load() { + try { + final List offers = objectMapper.readerForListOf(Offer.class).readValue(FILE); + log.info("Loaded {} offers from file: {}", offers.size(), FILE); + return offers; + } catch (IOException e) { + log.warn("Failed to load Offers from file={}: {}", FILE, e.toString()); + return new ArrayList<>(); + } + } + + private void save() { + try { + final List removed; + synchronized (offers) { + removed = _cleanUp(); + objectMapper.writerWithDefaultPrettyPrinter().writeValue(FILE, offers); + log.info("Wrote {} offers to file: {}", offers.size(), FILE); + } + removed.forEach(remove); + } catch (IOException e) { + log.warn("Failed to write Offers to file={}: {}", FILE, e.toString()); + } + } + + private List _cleanUp() { + if (offers.stream().anyMatch(Offer::_deleted_)) { + throw new RuntimeException(); + } + + offers.sort(Comparator.comparing(Offer::getDate)); + + final List deleted = new ArrayList<>(); + final List notToRemember = new ArrayList<>(offers.stream().filter(offer -> !offer.isRemember()).toList()); + while (!notToRemember.isEmpty() && notToRemember.size() > Kleinanzeigen.KEEP_LAST_OFFERS_COUNT) { + final Offer offer = notToRemember.removeFirst(); + offers.remove(offer); + offer.markDeleted(); + deleted.add(offer); + } + + return deleted; + } + + public void fetch() { + try { + final Document document = Jsoup.parse(VERSCHENKEN_EPPELBORN_30KM.toURL(), 3000); + for (Element article : document.select("li.ad-listitem:not(.is-topad) article.aditem")) { + try { + final Offer offer = parse(article); + merge(offer); + } catch (OfferParseException e) { + log.error("Failed to parse Offer:", e); + } + } + save(); + } catch (IOException e) { + log.error("Failed to fetch Kleinanzeigen: {}", e.toString()); + } + } + + private Offer parse(final Element article) throws OfferParseException { + try { + final String id = article.attr("data-adid"); + final String title = article.select(".text-module-begin").text(); + final String description = article.select(".aditem-main--middle--description").text(); + final ZonedDateTime date = parseDate(article.select(".aditem-main--top--right").text()); + final String articleURL = VERSCHENKEN_EPPELBORN_30KM.resolve(article.select(".aditem-image a").attr("href")).toString(); + final String imageURL = article.select(".aditem-image img").attr("src"); + final String zipcode; + final String location; + final Integer distance; + final String locationString = article.select(".aditem-main--top--left").text(); + final Matcher locationMatcher = Pattern.compile("^(?\\d+) (?.+) \\((:?ca.)?\\s*(?\\d+)\\s*km\\s*\\)$").matcher(locationString); + if (!locationMatcher.find()) { + zipcode = ""; + location = locationString; + distance = null; + } else { + zipcode = locationMatcher.group("zipcode"); + location = locationMatcher.group("location"); + distance = Integer.parseInt(locationMatcher.group("distance")); + } + return new Offer(id, date, title, zipcode, location, distance, description, articleURL, imageURL); + } catch (NumberFormatException e) { + throw new OfferParseException(article, e); + } + } + + private void merge(final Offer offer) { + synchronized (offer) { + offers.stream() + .filter(existing -> existing.getId().equals(offer.getId())) + .peek(existing -> { + existing.merge(offer); + log.info("Updated: {}", existing); + }) + .findFirst() + .orElseGet(() -> { + log.info("Created: {}", offer); + offers.add(offer); + return offer; + }); + } + } + + private ZonedDateTime parseDate(final String text) { + final Matcher dayNameMatcher = Pattern.compile("(?Gestern|Heute), (?\\d+):(?\\d+)").matcher(text); + if (dayNameMatcher.find()) { + final long minusDays = dayNameMatcher.group("day").equals("Gestern") ? 1 : 0; + return ZonedDateTime.now().minusDays(minusDays).withHour(Integer.parseInt(dayNameMatcher.group("hour"))).withMinute(Integer.parseInt(dayNameMatcher.group("minute"))).withSecond(0).withNano(0); + } + + final Matcher localDateMatcher = Pattern.compile("(?\\d+).(?\\d+).(?\\d+)").matcher(text); + if (localDateMatcher.find()) { + return ZonedDateTime.of( + LocalDate.of( + Integer.parseInt(localDateMatcher.group("day")), + Integer.parseInt(localDateMatcher.group("month")), + Integer.parseInt(localDateMatcher.group("year")) + ), + LocalTime.MIDNIGHT, + TimeZone.getDefault().toZoneId() + ); + } + throw new NumberFormatException("Failed to parse date: " + text); + } + + public List findAll() { + synchronized (offers) { + return new ArrayList<>(offers); + } + } + + public void ignore(final MaybeInaccessibleMessage message) { + synchronized (offers) { + findByTelegramMessageId(message).ifPresent(offer -> { + offer.ignore(); + save(); + }); + } + } + + public Optional remember(final MaybeInaccessibleMessage message, final boolean remember) { + synchronized (offers) { + final Optional optional = findByTelegramMessageId(message); + optional.ifPresent(offer -> { + offer.setRemember(remember); + save(); + }); + return optional; + } + } + + private Optional findByTelegramMessageId(final MaybeInaccessibleMessage message) { + return offers.stream().filter(offer -> Objects.equals(offer.getTelegramMessageId(), message.getMessageId())).findFirst(); + } + +} diff --git a/src/main/java/de/ph87/kleinanzeigen/api/Main.java b/src/main/java/de/ph87/kleinanzeigen/api/Main.java new file mode 100644 index 0000000..e0c784e --- /dev/null +++ b/src/main/java/de/ph87/kleinanzeigen/api/Main.java @@ -0,0 +1,43 @@ +package de.ph87.kleinanzeigen.api; + +import lombok.extern.slf4j.Slf4j; +import org.telegram.telegrambots.meta.exceptions.TelegramApiException; + +import java.io.IOException; +import java.util.List; + +@Slf4j +@SuppressWarnings({"InfiniteLoopStatement", "SameParameterValue", "SynchronizationOnLocalVariableOrMethodParameter"}) +public class Main { + + private static Bot bot; + + private static final Kleinanzeigen kleinanzeigen = new Kleinanzeigen(offer -> bot.remove(List.of(offer))); + + public static void main(String[] args) throws IOException, TelegramApiException { + bot = new Bot(kleinanzeigen::ignore, kleinanzeigen::remember); + try { + while (true) { + handle(bot); + waitSeconds(60); + } + } catch (InterruptedException e) { + log.warn(e.toString()); + } finally { + bot.stop(); + } + } + + private static void handle(final Bot bot) { + kleinanzeigen.fetch(); + kleinanzeigen.findAll().stream().filter(offer -> offer.getTelegramMessageId() == null).forEach(bot::send); + } + + private static void waitSeconds(final long seconds) throws InterruptedException { + final Object lock = new Object(); + synchronized (lock) { + lock.wait(seconds * 1000); + } + } + +} diff --git a/src/main/java/de/ph87/kleinanzeigen/api/Offer.java b/src/main/java/de/ph87/kleinanzeigen/api/Offer.java new file mode 100644 index 0000000..70d8a5a --- /dev/null +++ b/src/main/java/de/ph87/kleinanzeigen/api/Offer.java @@ -0,0 +1,130 @@ +package de.ph87.kleinanzeigen.api; + +import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.time.Duration; +import java.time.ZonedDateTime; + +@Getter +@NoArgsConstructor +public class Offer { + + private String id; + + @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX") + private ZonedDateTime first = ZonedDateTime.now(); + + @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX") + private ZonedDateTime last = first; + + @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX") + private ZonedDateTime date; + + private String title; + + private String zipcode; + + private String location; + + private Integer distance; + + private String description; + + private String href; + + private String image; + + @Setter + private Integer telegramMessageId = null; + + private boolean ignore = false; + + @Setter + private boolean remember = false; + + @JsonIgnore + private boolean _deleted_ = false; + + public Offer(final String id, final ZonedDateTime date, final String title, final String zipcode, final String location, final Integer distance, final String description, final String href, final String image) { + this.id = id; + this.date = date; + this.title = title; + this.zipcode = zipcode; + this.location = location; + this.distance = distance; + this.description = description; + this.href = href; + this.image = image; + } + + @Override + public String toString() { + return "Offer(%s%s, %s, %s)".formatted(_deleted_ ? "[DELETED], " : "", title, calculateLocationString(), href); + } + + public void verifyNotDeleted() { + if (_deleted_) { + throw new RuntimeException(); + } + } + + public String ageString() { + final Duration age = Duration.between(date, ZonedDateTime.now()); + if (age.toDays() > 0) { + return "%d Tag%s".formatted(age.toDays(), age.toDays() == 1 ? "" : "en"); + } else if (age.toHours() > 0) { + return "%d Stunde%s".formatted(age.toHours(), age.toHours() == 1 ? "" : "n"); + } else { + return "%d Minute%s".formatted(age.toMinutes(), age.toMinutes() == 1 ? "" : "n"); + } + } + + public void merge(final Offer other) { + verifyNotDeleted(); + other.verifyNotDeleted(); + if (!id.equals(other.id)) { + throw new RuntimeException(); + } + this.date = other.date; + this.title = other.title; + this.zipcode = other.zipcode; + this.location = other.location; + this.distance = other.distance; + this.description = other.description; + this.href = other.href; + this.image = other.image; + this.first = other.first.isBefore(this.first) ? other.first : this.first; + this.last = other.last.isAfter(this.last) ? other.last : this.last; + other.markDeleted(); + } + + public void markDeleted() { + _deleted_ = true; + } + + public boolean _deleted_() { + return _deleted_; + } + + public String calculateLocationString() { + String result = zipcode; + if (!result.isEmpty()) { + result += " "; + } + result += location; + if (distance != null) { + result += " (" + distance + " km)"; + } + return result; + } + + public void ignore() { + ignore = true; + remember = false; + } + +} diff --git a/src/main/java/de/ph87/kleinanzeigen/api/OfferParseException.java b/src/main/java/de/ph87/kleinanzeigen/api/OfferParseException.java new file mode 100644 index 0000000..1b70771 --- /dev/null +++ b/src/main/java/de/ph87/kleinanzeigen/api/OfferParseException.java @@ -0,0 +1,11 @@ +package de.ph87.kleinanzeigen.api; + +import org.jsoup.nodes.Element; + +public class OfferParseException extends Exception { + + public OfferParseException(final Element element, final RuntimeException cause) { + super(element.outerHtml(), cause); + } + +}