Kleinanzeigen + Telegram Bot working + Ignore,Remember
This commit is contained in:
commit
025c1d6560
42
.gitignore
vendored
Normal file
42
.gitignore
vendored
Normal file
@ -0,0 +1,42 @@
|
|||||||
|
/offers.json
|
||||||
|
/.idea/
|
||||||
|
/telegram.token
|
||||||
|
|
||||||
|
target/
|
||||||
|
!.mvn/wrapper/maven-wrapper.jar
|
||||||
|
!**/src/main/**/target/
|
||||||
|
!**/src/test/**/target/
|
||||||
|
|
||||||
|
### IntelliJ IDEA ###
|
||||||
|
.idea/modules.xml
|
||||||
|
.idea/jarRepositories.xml
|
||||||
|
.idea/compiler.xml
|
||||||
|
.idea/libraries/
|
||||||
|
*.iws
|
||||||
|
*.iml
|
||||||
|
*.ipr
|
||||||
|
|
||||||
|
### Eclipse ###
|
||||||
|
.apt_generated
|
||||||
|
.classpath
|
||||||
|
.factorypath
|
||||||
|
.project
|
||||||
|
.settings
|
||||||
|
.springBeans
|
||||||
|
.sts4-cache
|
||||||
|
|
||||||
|
### NetBeans ###
|
||||||
|
/nbproject/private/
|
||||||
|
/nbbuild/
|
||||||
|
/dist/
|
||||||
|
/nbdist/
|
||||||
|
/.nb-gradle/
|
||||||
|
build/
|
||||||
|
!**/src/main/**/build/
|
||||||
|
!**/src/test/**/build/
|
||||||
|
|
||||||
|
### VS Code ###
|
||||||
|
.vscode/
|
||||||
|
|
||||||
|
### Mac OS ###
|
||||||
|
.DS_Store
|
||||||
45
pom.xml
Normal file
45
pom.xml
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
<groupId>de.ph87</groupId>
|
||||||
|
<artifactId>Kleinanzeigen</artifactId>
|
||||||
|
<version>1.0-SNAPSHOT</version>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<maven.compiler.source>21</maven.compiler.source>
|
||||||
|
<maven.compiler.target>21</maven.compiler.target>
|
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.projectlombok</groupId>
|
||||||
|
<artifactId>lombok</artifactId>
|
||||||
|
<version>1.18.32</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.jsoup</groupId>
|
||||||
|
<artifactId>jsoup</artifactId>
|
||||||
|
<version>1.16.1</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.telegram</groupId>
|
||||||
|
<artifactId>telegrambots</artifactId>
|
||||||
|
<version>6.9.7.1</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-simple</artifactId>
|
||||||
|
<version>2.0.12</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.fasterxml.jackson.datatype</groupId>
|
||||||
|
<artifactId>jackson-datatype-jsr310</artifactId>
|
||||||
|
<version>2.15.4</version>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
</project>
|
||||||
198
src/main/java/de/ph87/kleinanzeigen/api/Bot.java
Normal file
198
src/main/java/de/ph87/kleinanzeigen/api/Bot.java
Normal file
@ -0,0 +1,198 @@
|
|||||||
|
package de.ph87.kleinanzeigen.api;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.telegram.telegrambots.bots.TelegramLongPollingBot;
|
||||||
|
import org.telegram.telegrambots.meta.TelegramBotsApi;
|
||||||
|
import org.telegram.telegrambots.meta.api.methods.send.SendMessage;
|
||||||
|
import org.telegram.telegrambots.meta.api.methods.send.SendPhoto;
|
||||||
|
import org.telegram.telegrambots.meta.api.methods.updatingmessages.DeleteMessages;
|
||||||
|
import org.telegram.telegrambots.meta.api.methods.updatingmessages.EditMessageCaption;
|
||||||
|
import org.telegram.telegrambots.meta.api.methods.updatingmessages.EditMessageText;
|
||||||
|
import org.telegram.telegrambots.meta.api.objects.*;
|
||||||
|
import org.telegram.telegrambots.meta.api.objects.replykeyboard.InlineKeyboardMarkup;
|
||||||
|
import org.telegram.telegrambots.meta.api.objects.replykeyboard.buttons.InlineKeyboardButton;
|
||||||
|
import org.telegram.telegrambots.meta.exceptions.TelegramApiException;
|
||||||
|
import org.telegram.telegrambots.updatesreceivers.DefaultBotSession;
|
||||||
|
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.*;
|
||||||
|
import java.util.function.BiFunction;
|
||||||
|
import java.util.function.Consumer;
|
||||||
|
|
||||||
|
import static de.ph87.kleinanzeigen.api.JSON.objectMapper;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
public class Bot extends TelegramLongPollingBot {
|
||||||
|
|
||||||
|
private static final long CHAT_ID = 101138682L;
|
||||||
|
|
||||||
|
private final DefaultBotSession session;
|
||||||
|
|
||||||
|
private final Consumer<MaybeInaccessibleMessage> ignore;
|
||||||
|
|
||||||
|
private final BiFunction<MaybeInaccessibleMessage, Boolean, Optional<Offer>> remember;
|
||||||
|
|
||||||
|
public Bot(final Consumer<MaybeInaccessibleMessage> ignore, final BiFunction<MaybeInaccessibleMessage, Boolean, Optional<Offer>> remember) throws IOException, TelegramApiException {
|
||||||
|
super(readToken());
|
||||||
|
this.ignore = ignore;
|
||||||
|
this.remember = remember;
|
||||||
|
log.info("Starting telegram bot...");
|
||||||
|
final TelegramBotsApi api = new TelegramBotsApi(DefaultBotSession.class);
|
||||||
|
session = (DefaultBotSession) api.registerBot(this);
|
||||||
|
log.info("Telegram bot registered.");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getBotUsername() {
|
||||||
|
return "BotKleinanzeigenBot";
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onUpdateReceived(final Update update) {
|
||||||
|
if (update.hasMessage() && update.getMessage().hasText()) {
|
||||||
|
log.info("#{} \"{}\": {}", update.getMessage().getChat().getId(), update.getMessage().getChat().getUserName(), update.getMessage().getText());
|
||||||
|
} else if (update.hasCallbackQuery()) {
|
||||||
|
handleCallbackQuery(update.getCallbackQuery());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void handleCallbackQuery(final CallbackQuery query) {
|
||||||
|
final MaybeInaccessibleMessage message = query.getMessage();
|
||||||
|
if (message.getChatId() != CHAT_ID) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
final InlineDto dto = objectMapper.readValue(query.getData(), InlineDto.class);
|
||||||
|
switch (dto.getCommand()) {
|
||||||
|
case IGNORE -> ignore(message);
|
||||||
|
case REMEMBER -> remember(message, dto);
|
||||||
|
case UNREMEMBER -> unremember(message, dto);
|
||||||
|
}
|
||||||
|
} catch (JsonProcessingException e) {
|
||||||
|
log.error("Failed to read InlineDto.", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ignore(final MaybeInaccessibleMessage message) {
|
||||||
|
ignore.accept(message);
|
||||||
|
remove(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void remember(final MaybeInaccessibleMessage message, final InlineDto dto) {
|
||||||
|
remember.apply(message, true).ifPresent(offer -> editMessage(message, offer, dto));
|
||||||
|
}
|
||||||
|
|
||||||
|
private void unremember(final MaybeInaccessibleMessage message, final InlineDto dto) {
|
||||||
|
remember.apply(message, false).ifPresent(offer -> editMessage(message, offer, dto));
|
||||||
|
}
|
||||||
|
|
||||||
|
private void editMessage(final MaybeInaccessibleMessage message, final Offer offer, final InlineDto dto) {
|
||||||
|
try {
|
||||||
|
switch (dto.getType()) {
|
||||||
|
case TEXT -> {
|
||||||
|
final EditMessageText edit = new EditMessageText(message.getChatId() + "", message.getMessageId(), null, getText(offer), null, null, getKeyboard(offer), null, null);
|
||||||
|
edit.setParseMode("Markdown");
|
||||||
|
execute(edit);
|
||||||
|
}
|
||||||
|
case PHOTO -> {
|
||||||
|
final EditMessageCaption edit = new EditMessageCaption(message.getChatId() + "", message.getMessageId(), null, getText(offer), getKeyboard(offer), null, null);
|
||||||
|
edit.setParseMode("Markdown");
|
||||||
|
execute(edit);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (TelegramApiException | JsonProcessingException e) {
|
||||||
|
log.error("Failed to edit Message to #{}.", CHAT_ID, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String readToken() throws IOException {
|
||||||
|
try (final FileInputStream stream = new FileInputStream("./telegram.token")) {
|
||||||
|
return new String(stream.readAllBytes(), StandardCharsets.UTF_8).trim();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void send(final Offer offer) {
|
||||||
|
try {
|
||||||
|
final Message message;
|
||||||
|
if (offer.getImage().isEmpty()) {
|
||||||
|
final SendMessage send = new SendMessage(CHAT_ID + "", getText(offer));
|
||||||
|
send.setParseMode("Markdown");
|
||||||
|
send.setReplyMarkup(getKeyboard(offer));
|
||||||
|
message = execute(send);
|
||||||
|
} else {
|
||||||
|
final SendPhoto send = new SendPhoto(CHAT_ID + "", new InputFile(offer.getImage()));
|
||||||
|
send.setCaption(getText(offer));
|
||||||
|
send.setParseMode("Markdown");
|
||||||
|
send.setReplyMarkup(getKeyboard(offer));
|
||||||
|
message = execute(send);
|
||||||
|
}
|
||||||
|
|
||||||
|
offer.setTelegramMessageId(message.getMessageId());
|
||||||
|
} catch (TelegramApiException | JsonProcessingException e) {
|
||||||
|
log.error("Failed to send Message to #{}.", CHAT_ID, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String getText(final Offer offer) {
|
||||||
|
return "[%s](%s)\nVor %s\n%s\n%s".formatted(
|
||||||
|
escape(offer.getTitle()),
|
||||||
|
offer.getHref(),
|
||||||
|
offer.ageString(),
|
||||||
|
offer.calculateLocationString(),
|
||||||
|
offer.getDescription()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String escape(final String text) {
|
||||||
|
// TODO make smarter (real escaping)
|
||||||
|
return text.replaceAll("\\[", "(").replaceAll("]", ")");
|
||||||
|
}
|
||||||
|
|
||||||
|
private InlineKeyboardMarkup getKeyboard(final Offer offer) throws JsonProcessingException {
|
||||||
|
final InlineKeyboardMarkup markup = new InlineKeyboardMarkup();
|
||||||
|
final ArrayList<List<InlineKeyboardButton>> keyboard = new ArrayList<>();
|
||||||
|
final ArrayList<InlineKeyboardButton> row = new ArrayList<>();
|
||||||
|
if (offer.isRemember()) {
|
||||||
|
addButton(row, "Nicht mehr merken", InlineCommand.UNREMEMBER, offer);
|
||||||
|
} else {
|
||||||
|
addButton(row, "Ignorieren", InlineCommand.IGNORE, offer);
|
||||||
|
addButton(row, "Merken", InlineCommand.REMEMBER, offer);
|
||||||
|
}
|
||||||
|
keyboard.add(row);
|
||||||
|
markup.setKeyboard(keyboard);
|
||||||
|
return markup;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addButton(final ArrayList<InlineKeyboardButton> row, final String caption, final InlineCommand command, final Offer offer) throws JsonProcessingException {
|
||||||
|
final InlineDto dto = new InlineDto(command, InlineType.of(offer));
|
||||||
|
final String data = objectMapper.writeValueAsString(dto);
|
||||||
|
row.add(new InlineKeyboardButton(caption, null, data, null, null, null, null, null, null));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void remove(final List<Offer> offers) {
|
||||||
|
_remove(offers.stream().map(Offer::getTelegramMessageId).filter(Objects::nonNull).toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void remove(final MaybeInaccessibleMessage... messages) {
|
||||||
|
_remove(Arrays.stream(messages).map(MaybeInaccessibleMessage::getMessageId).toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _remove(final List<Integer> messageIds) {
|
||||||
|
if (messageIds.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
execute(new DeleteMessages(CHAT_ID + "", messageIds));
|
||||||
|
} catch (TelegramApiException e) {
|
||||||
|
log.error("Failed to remove Message to #{}.", CHAT_ID, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void stop() {
|
||||||
|
session.stop();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,5 @@
|
|||||||
|
package de.ph87.kleinanzeigen.api;
|
||||||
|
|
||||||
|
public enum InlineCommand {
|
||||||
|
IGNORE, REMEMBER, UNREMEMBER
|
||||||
|
}
|
||||||
19
src/main/java/de/ph87/kleinanzeigen/api/InlineDto.java
Normal file
19
src/main/java/de/ph87/kleinanzeigen/api/InlineDto.java
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
package de.ph87.kleinanzeigen.api;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
import lombok.NoArgsConstructor;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
@NoArgsConstructor
|
||||||
|
public class InlineDto {
|
||||||
|
|
||||||
|
private InlineCommand command;
|
||||||
|
|
||||||
|
private InlineType type;
|
||||||
|
|
||||||
|
public InlineDto(final InlineCommand command, final InlineType type) {
|
||||||
|
this.command = command;
|
||||||
|
this.type = type;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
13
src/main/java/de/ph87/kleinanzeigen/api/InlineType.java
Normal file
13
src/main/java/de/ph87/kleinanzeigen/api/InlineType.java
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
package de.ph87.kleinanzeigen.api;
|
||||||
|
|
||||||
|
public enum InlineType {
|
||||||
|
TEXT, PHOTO;
|
||||||
|
|
||||||
|
public static InlineType of(final Offer offer) {
|
||||||
|
if (offer.getImage() == null) {
|
||||||
|
return TEXT;
|
||||||
|
}
|
||||||
|
return PHOTO;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
9
src/main/java/de/ph87/kleinanzeigen/api/JSON.java
Normal file
9
src/main/java/de/ph87/kleinanzeigen/api/JSON.java
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
package de.ph87.kleinanzeigen.api;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
|
public class JSON {
|
||||||
|
|
||||||
|
public static final ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules();
|
||||||
|
|
||||||
|
}
|
||||||
198
src/main/java/de/ph87/kleinanzeigen/api/Kleinanzeigen.java
Normal file
198
src/main/java/de/ph87/kleinanzeigen/api/Kleinanzeigen.java
Normal file
@ -0,0 +1,198 @@
|
|||||||
|
package de.ph87.kleinanzeigen.api;
|
||||||
|
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.jsoup.Jsoup;
|
||||||
|
import org.jsoup.nodes.Document;
|
||||||
|
import org.jsoup.nodes.Element;
|
||||||
|
import org.telegram.telegrambots.meta.api.objects.MaybeInaccessibleMessage;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.time.LocalDate;
|
||||||
|
import java.time.LocalTime;
|
||||||
|
import java.time.ZonedDateTime;
|
||||||
|
import java.util.*;
|
||||||
|
import java.util.function.Consumer;
|
||||||
|
import java.util.regex.Matcher;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
import static de.ph87.kleinanzeigen.api.JSON.objectMapper;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
public class Kleinanzeigen {
|
||||||
|
|
||||||
|
private static final int KEEP_LAST_OFFERS_COUNT = 50;
|
||||||
|
|
||||||
|
private static final File FILE = new File("./offers.json");
|
||||||
|
|
||||||
|
private static final URI VERSCHENKEN_EPPELBORN_30KM = URI.create("https://www.kleinanzeigen.de/s-zu-verschenken/66571/c192l339r30");
|
||||||
|
|
||||||
|
private final List<Offer> offers;
|
||||||
|
|
||||||
|
private final Consumer<Offer> remove;
|
||||||
|
|
||||||
|
public Kleinanzeigen(final Consumer<Offer> remove) {
|
||||||
|
this.remove = remove;
|
||||||
|
offers = load();
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Offer> load() {
|
||||||
|
try {
|
||||||
|
final List<Offer> offers = objectMapper.readerForListOf(Offer.class).readValue(FILE);
|
||||||
|
log.info("Loaded {} offers from file: {}", offers.size(), FILE);
|
||||||
|
return offers;
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.warn("Failed to load Offers from file={}: {}", FILE, e.toString());
|
||||||
|
return new ArrayList<>();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void save() {
|
||||||
|
try {
|
||||||
|
final List<Offer> removed;
|
||||||
|
synchronized (offers) {
|
||||||
|
removed = _cleanUp();
|
||||||
|
objectMapper.writerWithDefaultPrettyPrinter().writeValue(FILE, offers);
|
||||||
|
log.info("Wrote {} offers to file: {}", offers.size(), FILE);
|
||||||
|
}
|
||||||
|
removed.forEach(remove);
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.warn("Failed to write Offers to file={}: {}", FILE, e.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Offer> _cleanUp() {
|
||||||
|
if (offers.stream().anyMatch(Offer::_deleted_)) {
|
||||||
|
throw new RuntimeException();
|
||||||
|
}
|
||||||
|
|
||||||
|
offers.sort(Comparator.comparing(Offer::getDate));
|
||||||
|
|
||||||
|
final List<Offer> deleted = new ArrayList<>();
|
||||||
|
final List<Offer> notToRemember = new ArrayList<>(offers.stream().filter(offer -> !offer.isRemember()).toList());
|
||||||
|
while (!notToRemember.isEmpty() && notToRemember.size() > Kleinanzeigen.KEEP_LAST_OFFERS_COUNT) {
|
||||||
|
final Offer offer = notToRemember.removeFirst();
|
||||||
|
offers.remove(offer);
|
||||||
|
offer.markDeleted();
|
||||||
|
deleted.add(offer);
|
||||||
|
}
|
||||||
|
|
||||||
|
return deleted;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void fetch() {
|
||||||
|
try {
|
||||||
|
final Document document = Jsoup.parse(VERSCHENKEN_EPPELBORN_30KM.toURL(), 3000);
|
||||||
|
for (Element article : document.select("li.ad-listitem:not(.is-topad) article.aditem")) {
|
||||||
|
try {
|
||||||
|
final Offer offer = parse(article);
|
||||||
|
merge(offer);
|
||||||
|
} catch (OfferParseException e) {
|
||||||
|
log.error("Failed to parse Offer:", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
save();
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error("Failed to fetch Kleinanzeigen: {}", e.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Offer parse(final Element article) throws OfferParseException {
|
||||||
|
try {
|
||||||
|
final String id = article.attr("data-adid");
|
||||||
|
final String title = article.select(".text-module-begin").text();
|
||||||
|
final String description = article.select(".aditem-main--middle--description").text();
|
||||||
|
final ZonedDateTime date = parseDate(article.select(".aditem-main--top--right").text());
|
||||||
|
final String articleURL = VERSCHENKEN_EPPELBORN_30KM.resolve(article.select(".aditem-image a").attr("href")).toString();
|
||||||
|
final String imageURL = article.select(".aditem-image img").attr("src");
|
||||||
|
final String zipcode;
|
||||||
|
final String location;
|
||||||
|
final Integer distance;
|
||||||
|
final String locationString = article.select(".aditem-main--top--left").text();
|
||||||
|
final Matcher locationMatcher = Pattern.compile("^(?<zipcode>\\d+) (?<location>.+) \\((:?ca.)?\\s*(?<distance>\\d+)\\s*km\\s*\\)$").matcher(locationString);
|
||||||
|
if (!locationMatcher.find()) {
|
||||||
|
zipcode = "";
|
||||||
|
location = locationString;
|
||||||
|
distance = null;
|
||||||
|
} else {
|
||||||
|
zipcode = locationMatcher.group("zipcode");
|
||||||
|
location = locationMatcher.group("location");
|
||||||
|
distance = Integer.parseInt(locationMatcher.group("distance"));
|
||||||
|
}
|
||||||
|
return new Offer(id, date, title, zipcode, location, distance, description, articleURL, imageURL);
|
||||||
|
} catch (NumberFormatException e) {
|
||||||
|
throw new OfferParseException(article, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void merge(final Offer offer) {
|
||||||
|
synchronized (offer) {
|
||||||
|
offers.stream()
|
||||||
|
.filter(existing -> existing.getId().equals(offer.getId()))
|
||||||
|
.peek(existing -> {
|
||||||
|
existing.merge(offer);
|
||||||
|
log.info("Updated: {}", existing);
|
||||||
|
})
|
||||||
|
.findFirst()
|
||||||
|
.orElseGet(() -> {
|
||||||
|
log.info("Created: {}", offer);
|
||||||
|
offers.add(offer);
|
||||||
|
return offer;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private ZonedDateTime parseDate(final String text) {
|
||||||
|
final Matcher dayNameMatcher = Pattern.compile("(?<day>Gestern|Heute), (?<hour>\\d+):(?<minute>\\d+)").matcher(text);
|
||||||
|
if (dayNameMatcher.find()) {
|
||||||
|
final long minusDays = dayNameMatcher.group("day").equals("Gestern") ? 1 : 0;
|
||||||
|
return ZonedDateTime.now().minusDays(minusDays).withHour(Integer.parseInt(dayNameMatcher.group("hour"))).withMinute(Integer.parseInt(dayNameMatcher.group("minute"))).withSecond(0).withNano(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
final Matcher localDateMatcher = Pattern.compile("(?<day>\\d+).(?<month>\\d+).(?<year>\\d+)").matcher(text);
|
||||||
|
if (localDateMatcher.find()) {
|
||||||
|
return ZonedDateTime.of(
|
||||||
|
LocalDate.of(
|
||||||
|
Integer.parseInt(localDateMatcher.group("day")),
|
||||||
|
Integer.parseInt(localDateMatcher.group("month")),
|
||||||
|
Integer.parseInt(localDateMatcher.group("year"))
|
||||||
|
),
|
||||||
|
LocalTime.MIDNIGHT,
|
||||||
|
TimeZone.getDefault().toZoneId()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
throw new NumberFormatException("Failed to parse date: " + text);
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<Offer> findAll() {
|
||||||
|
synchronized (offers) {
|
||||||
|
return new ArrayList<>(offers);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void ignore(final MaybeInaccessibleMessage message) {
|
||||||
|
synchronized (offers) {
|
||||||
|
findByTelegramMessageId(message).ifPresent(offer -> {
|
||||||
|
offer.ignore();
|
||||||
|
save();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public Optional<Offer> remember(final MaybeInaccessibleMessage message, final boolean remember) {
|
||||||
|
synchronized (offers) {
|
||||||
|
final Optional<Offer> optional = findByTelegramMessageId(message);
|
||||||
|
optional.ifPresent(offer -> {
|
||||||
|
offer.setRemember(remember);
|
||||||
|
save();
|
||||||
|
});
|
||||||
|
return optional;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Optional<Offer> findByTelegramMessageId(final MaybeInaccessibleMessage message) {
|
||||||
|
return offers.stream().filter(offer -> Objects.equals(offer.getTelegramMessageId(), message.getMessageId())).findFirst();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
43
src/main/java/de/ph87/kleinanzeigen/api/Main.java
Normal file
43
src/main/java/de/ph87/kleinanzeigen/api/Main.java
Normal file
@ -0,0 +1,43 @@
|
|||||||
|
package de.ph87.kleinanzeigen.api;
|
||||||
|
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.telegram.telegrambots.meta.exceptions.TelegramApiException;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
|
@SuppressWarnings({"InfiniteLoopStatement", "SameParameterValue", "SynchronizationOnLocalVariableOrMethodParameter"})
|
||||||
|
public class Main {
|
||||||
|
|
||||||
|
private static Bot bot;
|
||||||
|
|
||||||
|
private static final Kleinanzeigen kleinanzeigen = new Kleinanzeigen(offer -> bot.remove(List.of(offer)));
|
||||||
|
|
||||||
|
public static void main(String[] args) throws IOException, TelegramApiException {
|
||||||
|
bot = new Bot(kleinanzeigen::ignore, kleinanzeigen::remember);
|
||||||
|
try {
|
||||||
|
while (true) {
|
||||||
|
handle(bot);
|
||||||
|
waitSeconds(60);
|
||||||
|
}
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
log.warn(e.toString());
|
||||||
|
} finally {
|
||||||
|
bot.stop();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void handle(final Bot bot) {
|
||||||
|
kleinanzeigen.fetch();
|
||||||
|
kleinanzeigen.findAll().stream().filter(offer -> offer.getTelegramMessageId() == null).forEach(bot::send);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void waitSeconds(final long seconds) throws InterruptedException {
|
||||||
|
final Object lock = new Object();
|
||||||
|
synchronized (lock) {
|
||||||
|
lock.wait(seconds * 1000);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
130
src/main/java/de/ph87/kleinanzeigen/api/Offer.java
Normal file
130
src/main/java/de/ph87/kleinanzeigen/api/Offer.java
Normal file
@ -0,0 +1,130 @@
|
|||||||
|
package de.ph87.kleinanzeigen.api;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.NoArgsConstructor;
|
||||||
|
import lombok.Setter;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.ZonedDateTime;
|
||||||
|
|
||||||
|
@Getter
|
||||||
|
@NoArgsConstructor
|
||||||
|
public class Offer {
|
||||||
|
|
||||||
|
private String id;
|
||||||
|
|
||||||
|
@JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX")
|
||||||
|
private ZonedDateTime first = ZonedDateTime.now();
|
||||||
|
|
||||||
|
@JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX")
|
||||||
|
private ZonedDateTime last = first;
|
||||||
|
|
||||||
|
@JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX")
|
||||||
|
private ZonedDateTime date;
|
||||||
|
|
||||||
|
private String title;
|
||||||
|
|
||||||
|
private String zipcode;
|
||||||
|
|
||||||
|
private String location;
|
||||||
|
|
||||||
|
private Integer distance;
|
||||||
|
|
||||||
|
private String description;
|
||||||
|
|
||||||
|
private String href;
|
||||||
|
|
||||||
|
private String image;
|
||||||
|
|
||||||
|
@Setter
|
||||||
|
private Integer telegramMessageId = null;
|
||||||
|
|
||||||
|
private boolean ignore = false;
|
||||||
|
|
||||||
|
@Setter
|
||||||
|
private boolean remember = false;
|
||||||
|
|
||||||
|
@JsonIgnore
|
||||||
|
private boolean _deleted_ = false;
|
||||||
|
|
||||||
|
public Offer(final String id, final ZonedDateTime date, final String title, final String zipcode, final String location, final Integer distance, final String description, final String href, final String image) {
|
||||||
|
this.id = id;
|
||||||
|
this.date = date;
|
||||||
|
this.title = title;
|
||||||
|
this.zipcode = zipcode;
|
||||||
|
this.location = location;
|
||||||
|
this.distance = distance;
|
||||||
|
this.description = description;
|
||||||
|
this.href = href;
|
||||||
|
this.image = image;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return "Offer(%s%s, %s, %s)".formatted(_deleted_ ? "[DELETED], " : "", title, calculateLocationString(), href);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void verifyNotDeleted() {
|
||||||
|
if (_deleted_) {
|
||||||
|
throw new RuntimeException();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public String ageString() {
|
||||||
|
final Duration age = Duration.between(date, ZonedDateTime.now());
|
||||||
|
if (age.toDays() > 0) {
|
||||||
|
return "%d Tag%s".formatted(age.toDays(), age.toDays() == 1 ? "" : "en");
|
||||||
|
} else if (age.toHours() > 0) {
|
||||||
|
return "%d Stunde%s".formatted(age.toHours(), age.toHours() == 1 ? "" : "n");
|
||||||
|
} else {
|
||||||
|
return "%d Minute%s".formatted(age.toMinutes(), age.toMinutes() == 1 ? "" : "n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void merge(final Offer other) {
|
||||||
|
verifyNotDeleted();
|
||||||
|
other.verifyNotDeleted();
|
||||||
|
if (!id.equals(other.id)) {
|
||||||
|
throw new RuntimeException();
|
||||||
|
}
|
||||||
|
this.date = other.date;
|
||||||
|
this.title = other.title;
|
||||||
|
this.zipcode = other.zipcode;
|
||||||
|
this.location = other.location;
|
||||||
|
this.distance = other.distance;
|
||||||
|
this.description = other.description;
|
||||||
|
this.href = other.href;
|
||||||
|
this.image = other.image;
|
||||||
|
this.first = other.first.isBefore(this.first) ? other.first : this.first;
|
||||||
|
this.last = other.last.isAfter(this.last) ? other.last : this.last;
|
||||||
|
other.markDeleted();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void markDeleted() {
|
||||||
|
_deleted_ = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean _deleted_() {
|
||||||
|
return _deleted_;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String calculateLocationString() {
|
||||||
|
String result = zipcode;
|
||||||
|
if (!result.isEmpty()) {
|
||||||
|
result += " ";
|
||||||
|
}
|
||||||
|
result += location;
|
||||||
|
if (distance != null) {
|
||||||
|
result += " (" + distance + " km)";
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void ignore() {
|
||||||
|
ignore = true;
|
||||||
|
remember = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,11 @@
|
|||||||
|
package de.ph87.kleinanzeigen.api;
|
||||||
|
|
||||||
|
import org.jsoup.nodes.Element;
|
||||||
|
|
||||||
|
public class OfferParseException extends Exception {
|
||||||
|
|
||||||
|
public OfferParseException(final Element element, final RuntimeException cause) {
|
||||||
|
super(element.outerHtml(), cause);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user