spring-boot WIP

This commit is contained in:
Patrick Haßel 2024-06-08 21:01:39 +02:00
parent 4caea1a80e
commit 74cc6a18d1
16 changed files with 680 additions and 571 deletions

1
.gitignore vendored
View File

@ -1,6 +1,7 @@
/offers.json /offers.json
/.idea/ /.idea/
/telegram.token /telegram.token
/*.db
target/ target/
!.mvn/wrapper/maven-wrapper.jar !.mvn/wrapper/maven-wrapper.jar

9
application.properties Normal file
View File

@ -0,0 +1,9 @@
logging.level.de.ph87=DEBUG
#-
spring.datasource.url=jdbc:h2:./database;AUTO_SERVER=TRUE
spring.datasource.driverClassName=org.h2.Driver
spring.datasource.username=sa
spring.datasource.password=password
spring.jpa.database-platform=org.hibernate.dialect.H2Dialect
#-
#spring.jpa.hibernate.ddl-auto=create

70
pom.xml
View File

@ -14,65 +14,45 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties> </properties>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>3.3.0</version>
</parent>
<dependencies> <dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-json</artifactId>
<version>3.3.0</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
</dependency>
<dependency> <dependency>
<groupId>org.projectlombok</groupId> <groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId> <artifactId>lombok</artifactId>
<version>1.18.32</version> </dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
<version>2.3.1</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.jsoup</groupId> <groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId> <artifactId>jsoup</artifactId>
<version>1.16.1</version> <version>1.17.2</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.telegram</groupId> <groupId>org.telegram</groupId>
<artifactId>telegrambots</artifactId> <artifactId>telegrambots</artifactId>
<version>6.9.7.1</version> <version>6.9.7.1</version>
</dependency> </dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>2.0.12</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
<version>2.15.4</version>
</dependency>
</dependencies> </dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.6.0</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<version>3.3.0</version>
<configuration>
<archive>
<manifest>
<mainClass>de.ph87.kleinanzeigen.Main</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
</plugins>
</build>
</project> </project>

View File

@ -0,0 +1,17 @@
package de.ph87.kleinanzeigen;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@Slf4j
@SpringBootApplication
@RequiredArgsConstructor
public class Application {
public static void main(String[] args) {
SpringApplication.run(Application.class, args);
}
}

View File

@ -1,9 +0,0 @@
package de.ph87.kleinanzeigen;
import com.fasterxml.jackson.databind.ObjectMapper;
public class JSON {
public static final ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules();
}

View File

@ -1,48 +0,0 @@
package de.ph87.kleinanzeigen;
import de.ph87.kleinanzeigen.kleinanzeigen.KleinanzeigenApi;
import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferRepository;
import de.ph87.kleinanzeigen.telegram.TelegramBot;
import lombok.extern.slf4j.Slf4j;
import org.telegram.telegrambots.meta.exceptions.TelegramApiException;
import java.io.IOException;
import java.util.List;
@Slf4j
@SuppressWarnings({"InfiniteLoopStatement", "SameParameterValue", "SynchronizationOnLocalVariableOrMethodParameter"})
public class Main {
private static TelegramBot telegramBot;
private static final OfferRepository offerRepository = new OfferRepository(offer -> telegramBot.remove(List.of(offer)));
private static final KleinanzeigenApi kleinanzeigenApi = new KleinanzeigenApi(offerRepository);
public static void main(String[] args) throws IOException, TelegramApiException {
telegramBot = new TelegramBot(offerRepository);
try {
while (true) {
handle(telegramBot);
waitSeconds(60);
}
} catch (InterruptedException e) {
log.warn(e.toString());
} finally {
telegramBot.stop();
}
}
private static void handle(final TelegramBot telegramBot) {
kleinanzeigenApi.fetchUntilDuplicate(5);
offerRepository.findAll().stream().filter(offer -> offer.getTelegramMessageId() == null).forEach(telegramBot::send);
}
private static void waitSeconds(final long seconds) throws InterruptedException {
final Object lock = new Object();
synchronized (lock) {
lock.wait(seconds * 1000);
}
}
}

View File

@ -1,128 +1,75 @@
package de.ph87.kleinanzeigen.kleinanzeigen; package de.ph87.kleinanzeigen.kleinanzeigen;
import de.ph87.kleinanzeigen.kleinanzeigen.offer.Offer; import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferCreate;
import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferParseException; import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferDto;
import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferRepository; import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI; import java.net.URI;
import java.time.LocalDate; import java.util.concurrent.TimeUnit;
import java.time.LocalTime;
import java.time.ZonedDateTime;
import java.util.TimeZone;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Slf4j @Slf4j
@Service
@EnableScheduling
@RequiredArgsConstructor
public class KleinanzeigenApi { public class KleinanzeigenApi {
private static final String VERSCHENKEN_EPPELBORN_30KM = "https://www.kleinanzeigen.de/s-zu-verschenken/66571/seite:%d/c192l339r30"; private static final String VERSCHENKEN_EPPELBORN_30KM = "https://www.kleinanzeigen.de/s-zu-verschenken/66571/seite:%d/c192l339r30";
private final OfferRepository offerRepository; private static final int FETCH_UNTIL_DUPLICATE_MAX_PAGES = 1;
public KleinanzeigenApi(final OfferRepository offerRepository) { private final OfferService offerService;
this.offerRepository = offerRepository;
private final ApplicationEventPublisher publisher;
@Scheduled(initialDelay = 0, fixedRate = 1, timeUnit = TimeUnit.MINUTES)
public void fetch() {
fetchPagesUntilDuplicate();
} }
public void fetchUntilDuplicate(final int maxPageCount) { private void fetchPagesUntilDuplicate() {
int page = 0; int page = 0;
final FetchResult totalFetchResult = new FetchResult(); final FetchResult totalFetchResult = new FetchResult();
while (totalFetchResult.getUpdated() <= 0 && page <= maxPageCount) { while (totalFetchResult.getUpdated() <= 0 && page < FETCH_UNTIL_DUPLICATE_MAX_PAGES) {
final FetchResult pageFetchResult = fetch(++page); final FetchResult pageFetchResult = fetchPage(++page);
totalFetchResult.merge(pageFetchResult); totalFetchResult.merge(pageFetchResult);
} }
log.debug("FetchResult: {}", totalFetchResult); log.debug("FetchResult: {}", totalFetchResult);
} }
private FetchResult fetch(final int page) { private FetchResult fetchPage(final int page) {
final FetchResult fetchResult = new FetchResult(); final FetchResult fetchResult = new FetchResult();
final Document document;
final URI uri = URI.create(VERSCHENKEN_EPPELBORN_30KM.formatted(page));
try { try {
final URI uri = getPageURI(page);
log.debug("Fetching page: {}", uri); log.debug("Fetching page: {}", uri);
final Document document = Jsoup.parse(uri.toURL(), 3000); document = Jsoup.parse(uri.toURL(), 3000);
for (Element article : document.select("li.ad-listitem:not(.is-topad) article.aditem")) {
final Offer offer;
try {
offer = parse(article, uri);
} catch (OfferParseException e) {
log.error("Failed to parse Offer:", e);
fetchResult.add(MergeResult.ERROR);
continue;
}
final MergeResult mergeResult = offerRepository.save(offer);
fetchResult.add(mergeResult);
}
offerRepository.flush();
} catch (IOException e) { } catch (IOException e) {
log.error("Failed to fetch Kleinanzeigen: {}", e.toString()); log.error("Failed to fetch Kleinanzeigen: {}", e.toString());
return fetchResult;
} }
document.select("li.ad-listitem:not(.is-topad) article.aditem").forEach(article -> tryParse(article, uri, fetchResult));
return fetchResult; return fetchResult;
} }
private URI getPageURI(final int page) throws MalformedURLException { private void tryParse(final Element article, final URI uri, final FetchResult fetchResult) {
return URI.create(VERSCHENKEN_EPPELBORN_30KM.formatted(page));
}
private Offer parse(final Element article, final URI uri) throws OfferParseException {
try { try {
final String id = article.attr("data-adid"); final OfferCreate create = new OfferCreate(article, uri);
final String title = article.select(".text-module-begin").text(); final OfferDto dto = offerService.updateOrCreate(create, fetchResult);
final String description = article.select(".aditem-main--middle--description").text(); publisher.publishEvent(dto);
final ZonedDateTime date = parseDate(article.select(".aditem-main--top--right").text());
final String articleURL = uri.resolve(article.select(".aditem-image a").attr("href")).toString();
final String zipcode;
final String location;
final Integer distance;
final String locationString = article.select(".aditem-main--top--left").text();
final Matcher locationMatcher = Pattern.compile("^(?<zipcode>\\d+) (?<location>.+) \\((:?ca.)?\\s*(?<distance>\\d+)\\s*km\\s*\\)$").matcher(locationString);
if (!locationMatcher.find()) {
zipcode = "";
location = locationString;
distance = null;
} else {
zipcode = locationMatcher.group("zipcode");
location = locationMatcher.group("location");
distance = Integer.parseInt(locationMatcher.group("distance"));
}
final String imageURL = getImageURL(articleURL);
return new Offer(id, date, title, zipcode, location, distance, description, articleURL, imageURL);
} catch (NumberFormatException e) { } catch (NumberFormatException e) {
throw new OfferParseException(article, e); log.error("Failed to parse Offer:\n{}\n", article.outerHtml(), e);
fetchResult.add(MergeResult.ERROR);
} }
} }
private String getImageURL(final String articleURL) {
try {
final Document document = Jsoup.parse(URI.create(articleURL).toURL(), 3000);
final Element image = document.select(".galleryimage-element img").first();
if (image != null) {
return image.attr("src");
}
} catch (IOException e) {
log.error("Failed to load Article page: {}", articleURL);
}
return "";
}
private ZonedDateTime parseDate(final String text) {
final Matcher dayNameMatcher = Pattern.compile("(?<day>Gestern|Heute), (?<hour>\\d+):(?<minute>\\d+)").matcher(text);
if (dayNameMatcher.find()) {
final long minusDays = dayNameMatcher.group("day").equals("Gestern") ? 1 : 0;
return ZonedDateTime.now().minusDays(minusDays).withHour(Integer.parseInt(dayNameMatcher.group("hour"))).withMinute(Integer.parseInt(dayNameMatcher.group("minute"))).withSecond(0).withNano(0);
}
final Matcher localDateMatcher = Pattern.compile("(?<day>\\d+).(?<month>\\d+).(?<year>\\d+)").matcher(text);
if (localDateMatcher.find()) {
return ZonedDateTime.of(LocalDate.of(Integer.parseInt(localDateMatcher.group("day")), Integer.parseInt(localDateMatcher.group("month")), Integer.parseInt(localDateMatcher.group("year"))), LocalTime.MIDNIGHT, TimeZone.getDefault().toZoneId());
}
throw new NumberFormatException("Failed to parse date: " + text);
}
} }

View File

@ -1,131 +1,110 @@
package de.ph87.kleinanzeigen.kleinanzeigen.offer; package de.ph87.kleinanzeigen.kleinanzeigen.offer;
import com.fasterxml.jackson.annotation.JsonFormat; import jakarta.annotation.Nullable;
import com.fasterxml.jackson.annotation.JsonIgnore; import jakarta.persistence.*;
import lombok.Getter; import lombok.*;
import lombok.NoArgsConstructor;
import lombok.Setter;
import java.time.ZonedDateTime; import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.List;
@Entity
@Getter @Getter
@ToString(onlyExplicitlyIncluded = true)
@NoArgsConstructor @NoArgsConstructor
public class Offer { public class Offer {
private String id; @Id
@GeneratedValue
private long id;
@JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX") @Version
private long version;
@NonNull
@Column(nullable = false)
private ZonedDateTime first = ZonedDateTime.now(); private ZonedDateTime first = ZonedDateTime.now();
@JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX") @NonNull
@Column(nullable = false)
private ZonedDateTime last = first; private ZonedDateTime last = first;
@JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX") @NonNull
private ZonedDateTime date; @Column(nullable = false)
private ZonedDateTime expiry = first.plusDays(3);
private String title;
private String zipcode;
private String location;
private Integer distance;
private String description;
private String href;
private String image;
@Setter @Setter
private Integer telegramMessageId = null; @Column
private boolean hide = false;
private boolean ignore = false;
@Column
private boolean remember = false; private boolean remember = false;
@NonNull
@Column(nullable = false)
private String articleId;
@NonNull
@Column(nullable = false)
private ZonedDateTime articleDate;
@NonNull
@Column(nullable = false)
private String title;
@NonNull
@Column(nullable = false)
private String location;
@Column
@Nullable
private String zipcode = null;
@Column
@Nullable
private Integer distance = null;
@NonNull
@Column(nullable = false)
private String description;
@NonNull
@Column(nullable = false)
private String articleURL;
@Column
@Nullable
private String imageURL = null;
@Setter @Setter
@JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX") @Column
private ZonedDateTime rememberUntil = null; @Nullable
private Integer telegramMessageId = null;
@JsonIgnore public Offer(final @NonNull OfferCreate create) {
private boolean _deleted_ = false; this.articleId = create.getArticleId();
this.articleDate = create.getArticleDate();
public Offer(final String id, final ZonedDateTime date, final String title, final String zipcode, final String location, final Integer distance, final String description, final String href, final String image) { update(create);
this.id = id;
this.date = date;
this.title = title;
this.zipcode = zipcode;
this.location = location;
this.distance = distance;
this.description = description;
this.href = href;
this.image = image;
} }
@Override public void update(final OfferCreate create) {
public String toString() { this.title = create.getTitle();
return "Offer(%s%s, %s, %s)".formatted(_deleted_ ? "[DELETED], " : "", title, calculateLocationString(), href); this.zipcode = create.getZipcode();
this.location = create.getLocation();
this.distance = create.getDistance();
this.description = create.getDescription();
this.articleURL = create.getArticleURL();
this.imageURL = create.getImageURL();
} }
public void verifyNotDeleted() { public void setRemember(final boolean newRemember) {
if (_deleted_) { if (remember && !newRemember) {
throw new RuntimeException(); final ZonedDateTime oneHour = ZonedDateTime.now().plusHours(1);
if (oneHour.isAfter(expiry)) {
expiry = oneHour;
}
} }
} remember = newRemember;
public void merge(final Offer other) {
verifyNotDeleted();
other.verifyNotDeleted();
if (!id.equals(other.id)) {
throw new RuntimeException();
}
this.date = other.date;
this.title = other.title;
this.zipcode = other.zipcode;
this.location = other.location;
this.distance = other.distance;
this.description = other.description;
this.href = other.href;
this.image = other.image;
this.first = other.first.isBefore(this.first) ? other.first : this.first;
this.last = other.last.isAfter(this.last) ? other.last : this.last;
other.markDeleted();
}
public void markDeleted() {
_deleted_ = true;
}
public boolean _deleted_() {
return _deleted_;
}
public String calculateLocationString() {
String result = zipcode;
if (!result.isEmpty()) {
result += " ";
}
result += location;
if (distance != null) {
result += " (" + distance + " km)";
}
return result;
}
public void ignore() {
ignore = true;
remember = false;
rememberUntil = null;
}
public void setRemember(final boolean remember) {
if (remember) {
this.rememberUntil = null;
} else if (this.remember) {
this.rememberUntil = ZonedDateTime.now().plusHours(1);
}
this.remember = remember;
} }
} }

View File

@ -0,0 +1,103 @@
package de.ph87.kleinanzeigen.kleinanzeigen.offer;
import jakarta.annotation.Nullable;
import lombok.Getter;
import lombok.NonNull;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import java.io.IOException;
import java.net.URI;
import java.time.LocalDate;
import java.time.LocalTime;
import java.time.ZonedDateTime;
import java.util.TimeZone;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Slf4j
@Getter
@ToString(onlyExplicitlyIncluded = true)
public class OfferCreate {
@NonNull
@ToString.Include
private final String articleId;
@NonNull
private final ZonedDateTime articleDate;
@NonNull
@ToString.Include
private final String title;
@NonNull
@ToString.Include
private final String location;
@Nullable
private final String zipcode;
@Nullable
private final Integer distance;
@NonNull
private final String description;
@NonNull
private final String articleURL;
@Nullable
private final String imageURL;
public OfferCreate(final Element article, final URI uri) {
articleId = article.attr("data-adid");
title = article.select(".text-module-begin").text();
description = article.select(".aditem-main--middle--description").text();
articleDate = parseDate(article.select(".aditem-main--top--right").text());
articleURL = uri.resolve(article.select(".aditem-image a").attr("href")).toString();
final String locationString = article.select(".aditem-main--top--left").text();
final Matcher locationMatcher = Pattern.compile("^(?<zipcode>\\d+) (?<location>.+) \\((:?ca.)?\\s*(?<distance>\\d+)\\s*km\\s*\\)$").matcher(locationString);
if (!locationMatcher.find()) {
zipcode = "";
location = locationString;
distance = null;
} else {
zipcode = locationMatcher.group("zipcode");
location = locationMatcher.group("location");
distance = Integer.parseInt(locationMatcher.group("distance"));
}
imageURL = getImageURL(articleURL);
}
private ZonedDateTime parseDate(final String text) {
final Matcher dayNameMatcher = Pattern.compile("(?<day>Gestern|Heute), (?<hour>\\d+):(?<minute>\\d+)").matcher(text);
if (dayNameMatcher.find()) {
final long minusDays = dayNameMatcher.group("day").equals("Gestern") ? 1 : 0;
return ZonedDateTime.now().minusDays(minusDays).withHour(Integer.parseInt(dayNameMatcher.group("hour"))).withMinute(Integer.parseInt(dayNameMatcher.group("minute"))).withSecond(0).withNano(0);
}
final Matcher localDateMatcher = Pattern.compile("(?<day>\\d+).(?<month>\\d+).(?<year>\\d+)").matcher(text);
if (localDateMatcher.find()) {
return ZonedDateTime.of(LocalDate.of(Integer.parseInt(localDateMatcher.group("day")), Integer.parseInt(localDateMatcher.group("month")), Integer.parseInt(localDateMatcher.group("year"))), LocalTime.MIDNIGHT, TimeZone.getDefault().toZoneId());
}
throw new NumberFormatException("Failed to parse date: " + text);
}
private String getImageURL(final String articleURL) {
try {
final Document document = Jsoup.parse(URI.create(articleURL).toURL(), 3000);
final Element image = document.select(".galleryimage-element img").first();
if (image != null) {
return image.attr("src");
}
} catch (IOException e) {
log.error("Failed to load Article page: {}", articleURL);
}
return null;
}
}

View File

@ -0,0 +1,100 @@
package de.ph87.kleinanzeigen.kleinanzeigen.offer;
import jakarta.annotation.Nullable;
import lombok.Getter;
import lombok.NonNull;
import lombok.ToString;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.List;
@Getter
@ToString(onlyExplicitlyIncluded = true)
public class OfferDto {
private final long id;
private final long version;
@NonNull
private final ZonedDateTime first;
@NonNull
private final ZonedDateTime last;
@NonNull
private final ZonedDateTime expiry;
private final boolean hide;
private final boolean remember;
@NonNull
private final String articleId;
@NonNull
private final ZonedDateTime articleDate;
@NonNull
private final String title;
@NonNull
private final String location;
@Nullable
private final String zipcode;
@Nullable
private final Integer distance;
@NonNull
private final String description;
@NonNull
private final String articleURL;
@Nullable
private final String imageURL;
@Nullable
private final Integer telegramMessageId;
private final boolean _existing_;
public OfferDto(final @NonNull Offer offer, final boolean existing) {
id = offer.getId();
version = offer.getVersion();
first = offer.getFirst();
last = offer.getLast();
expiry = offer.getExpiry();
hide = offer.isHide();
remember = offer.isRemember();
articleId = offer.getArticleId();
articleDate = offer.getArticleDate();
title = offer.getTitle();
zipcode = offer.getZipcode();
location = offer.getLocation();
distance = offer.getDistance();
description = offer.getDescription();
articleURL = offer.getArticleURL();
imageURL = offer.getImageURL();
telegramMessageId = offer.getTelegramMessageId();
_existing_ = existing;
}
public String combineLocation() {
final List<String> list = new ArrayList<>();
if (zipcode != null) {
list.add(zipcode);
}
list.add(location);
if (distance != null) {
list.add("(" + distance + " km)");
}
return String.join(" ", list);
}
}

View File

@ -1,122 +1,17 @@
package de.ph87.kleinanzeigen.kleinanzeigen.offer; package de.ph87.kleinanzeigen.kleinanzeigen.offer;
import de.ph87.kleinanzeigen.kleinanzeigen.MergeResult; import org.springframework.data.repository.ListCrudRepository;
import lombok.extern.slf4j.Slf4j;
import org.telegram.telegrambots.meta.api.objects.MaybeInaccessibleMessage;
import java.io.File;
import java.io.IOException;
import java.time.ZonedDateTime; import java.time.ZonedDateTime;
import java.util.*; import java.util.List;
import java.util.function.Consumer; import java.util.Optional;
import static de.ph87.kleinanzeigen.JSON.objectMapper; public interface OfferRepository extends ListCrudRepository<Offer, Long> {
@Slf4j List<Offer> findAllByExpiryBefore(final ZonedDateTime deadline);
public class OfferRepository {
private static final int KEEP_LAST_OFFERS_COUNT = 200; Optional<Offer> findByArticleId(String articleId);
private static final File FILE = new File("./offers.json"); Optional<Offer> findByTelegramMessageId(int telegramMessageId);
private final List<Offer> offers;
private final Consumer<Offer> remove;
public OfferRepository(final Consumer<Offer> remove) {
this.remove = remove;
offers = load();
}
private List<Offer> load() {
try {
final List<Offer> offers = objectMapper.readerForListOf(Offer.class).readValue(FILE);
log.info("Loaded {} offers from file: {}", offers.size(), FILE);
return offers;
} catch (IOException e) {
log.warn("Failed to load Offers from file={}: {}", FILE, e.toString());
return new ArrayList<>();
}
}
public void flush() {
try {
final List<Offer> removed;
synchronized (offers) {
removed = _cleanUp();
objectMapper.writerWithDefaultPrettyPrinter().writeValue(FILE, offers);
log.debug("Wrote {} offers to file: {}", offers.size(), FILE);
}
removed.forEach(remove);
} catch (IOException e) {
log.warn("Failed to write Offers to file={}: {}", FILE, e.toString());
}
}
private List<Offer> _cleanUp() {
if (offers.stream().anyMatch(Offer::_deleted_)) {
throw new RuntimeException();
}
offers.sort(Comparator.comparing(Offer::getDate));
final ZonedDateTime now = ZonedDateTime.now();
final List<Offer> deleted = new ArrayList<>();
final List<Offer> removable = new ArrayList<>(offers.stream().filter(offer -> !offer.isRemember() && (offer.getRememberUntil() == null || now.isAfter(offer.getRememberUntil()))).toList());
while (!removable.isEmpty() && removable.size() > OfferRepository.KEEP_LAST_OFFERS_COUNT) {
final Offer offer = removable.removeFirst();
offers.remove(offer);
offer.markDeleted();
deleted.add(offer);
}
return deleted;
}
public MergeResult save(final Offer offer) {
synchronized (offer) {
final Optional<Offer> existingOptional = offers.stream().filter(existing -> existing.getId().equals(offer.getId())).findFirst();
if (existingOptional.isPresent()) {
existingOptional.get().merge(offer);
return MergeResult.UPDATED;
} else {
log.info("Created: {}", offer);
offers.add(offer);
return MergeResult.CREATED;
}
}
}
public List<Offer> findAll() {
synchronized (offers) {
return new ArrayList<>(offers);
}
}
public void ignore(final MaybeInaccessibleMessage message) {
synchronized (offers) {
findByTelegramMessageId(message).ifPresent(offer -> {
offer.ignore();
flush();
});
}
}
public Optional<Offer> remember(final MaybeInaccessibleMessage message, final boolean remember) {
synchronized (offers) {
final Optional<Offer> optional = findByTelegramMessageId(message);
optional.ifPresent(offer -> {
offer.setRemember(remember);
flush();
});
return optional;
}
}
public Optional<Offer> findByTelegramMessageId(final MaybeInaccessibleMessage message) {
synchronized (offers) {
return offers.stream().filter(offer -> Objects.equals(offer.getTelegramMessageId(), message.getMessageId())).findFirst();
}
}
} }

View File

@ -0,0 +1,73 @@
package de.ph87.kleinanzeigen.kleinanzeigen.offer;
import de.ph87.kleinanzeigen.kleinanzeigen.FetchResult;
import de.ph87.kleinanzeigen.kleinanzeigen.MergeResult;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.time.ZonedDateTime;
import java.util.List;
import java.util.Optional;
@Slf4j
@Service
@Transactional
@RequiredArgsConstructor
public class OfferService {
private final OfferRepository repository;
public List<OfferDto> cleanUp() {
final List<Offer> list = repository.findAllByExpiryBefore(ZonedDateTime.now());
repository.deleteAll(list);
return list.stream().map(offer -> toDto(offer, false)).toList();
}
private OfferDto toDto(final Offer offer, final boolean existing) {
return new OfferDto(offer, existing);
}
public OfferDto updateOrCreate(final OfferCreate create, final FetchResult fetchResult) {
return toDto(_updateOrCreate(create, fetchResult), true);
}
private Offer _updateOrCreate(final OfferCreate create, final FetchResult fetchResult) {
return repository
.findByArticleId(create.getArticleId())
.stream().peek(
existing -> {
existing.update(create);
fetchResult.add(MergeResult.UPDATED);
})
.findFirst()
.orElseGet(() -> {
fetchResult.add(MergeResult.CREATED);
return repository.save(new Offer(create));
}
);
}
@SuppressWarnings("UnusedReturnValue")
public Optional<OfferDto> hideByTelegramMessageId(final int messageId, final boolean hide) {
return repository.findByTelegramMessageId(messageId).stream().peek(offer -> offer.setHide(hide)).findFirst().map(offer -> toDto(offer, true));
}
public Optional<OfferDto> rememberByTelegramMessageId(final int messageId, final boolean remember) {
return repository.findByTelegramMessageId(messageId).stream().peek(offer -> offer.setRemember(remember)).findFirst().map(offer -> toDto(offer, true));
}
public Optional<OfferDto> findByTelegramMessageId(final int messageId) {
return repository.findByTelegramMessageId(messageId).map(offer -> toDto(offer, true));
}
public void setTelegramMessageId(final OfferDto dto, final Integer telegramMessageId) {
findByDto(dto).ifPresent(offer -> offer.setTelegramMessageId(telegramMessageId));
}
private Optional<Offer> findByDto(final OfferDto dto) {
return repository.findById(dto.getId());
}
}

View File

@ -1,5 +1,5 @@
package de.ph87.kleinanzeigen.telegram; package de.ph87.kleinanzeigen.telegram;
public enum InlineCommand { public enum InlineCommand {
IGNORE, REMEMBER, UNREMEMBER HIDE, REMEMBER, UNREMEMBER
} }

View File

@ -1,57 +1,25 @@
package de.ph87.kleinanzeigen.telegram; package de.ph87.kleinanzeigen.telegram;
import com.fasterxml.jackson.core.JsonProcessingException;
import de.ph87.kleinanzeigen.kleinanzeigen.offer.Offer;
import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferRepository;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.telegram.telegrambots.bots.TelegramLongPollingBot; import org.telegram.telegrambots.bots.TelegramLongPollingBot;
import org.telegram.telegrambots.meta.TelegramBotsApi; import org.telegram.telegrambots.meta.TelegramBotsApi;
import org.telegram.telegrambots.meta.api.methods.send.SendPhoto; import org.telegram.telegrambots.meta.api.objects.Update;
import org.telegram.telegrambots.meta.api.methods.updatingmessages.DeleteMessages;
import org.telegram.telegrambots.meta.api.methods.updatingmessages.EditMessageCaption;
import org.telegram.telegrambots.meta.api.objects.*;
import org.telegram.telegrambots.meta.api.objects.replykeyboard.InlineKeyboardMarkup;
import org.telegram.telegrambots.meta.api.objects.replykeyboard.buttons.InlineKeyboardButton;
import org.telegram.telegrambots.meta.exceptions.TelegramApiException; import org.telegram.telegrambots.meta.exceptions.TelegramApiException;
import org.telegram.telegrambots.updatesreceivers.DefaultBotSession; import org.telegram.telegrambots.updatesreceivers.DefaultBotSession;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.StandardCharsets; import java.util.function.Consumer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import static de.ph87.kleinanzeigen.JSON.objectMapper;
@Slf4j @Slf4j
public class TelegramBot extends TelegramLongPollingBot { public class TelegramBot extends TelegramLongPollingBot {
private static final long CHAT_ID = 101138682L;
private static final String ICON_CHECK = "";
private static final String ICON_REMOVE = "";
private final byte[] NO_IMAGE;
private final DefaultBotSession session; private final DefaultBotSession session;
private final OfferRepository offerRepository; private final Consumer<Update> onUpdate;
public TelegramBot(final OfferRepository offerRepository) throws IOException, TelegramApiException { public TelegramBot(final String token, final Consumer<Update> onUpdate) throws IOException, TelegramApiException {
super(readToken()); super(token);
this.offerRepository = offerRepository; this.onUpdate = onUpdate;
final BufferedImage img = new BufferedImage(1, 1, BufferedImage.TYPE_INT_ARGB);
final ByteArrayOutputStream stream = new ByteArrayOutputStream();
ImageIO.write(img, "PNG", stream);
NO_IMAGE = stream.toByteArray();
log.info("Starting telegram bot..."); log.info("Starting telegram bot...");
final TelegramBotsApi api = new TelegramBotsApi(DefaultBotSession.class); final TelegramBotsApi api = new TelegramBotsApi(DefaultBotSession.class);
@ -59,12 +27,6 @@ public class TelegramBot extends TelegramLongPollingBot {
log.info("Telegram bot registered."); log.info("Telegram bot registered.");
} }
private static String readToken() throws IOException {
try (final FileInputStream stream = new FileInputStream("./telegram.token")) {
return new String(stream.readAllBytes(), StandardCharsets.UTF_8).trim();
}
}
@Override @Override
public String getBotUsername() { public String getBotUsername() {
return "BotKleinanzeigenBot"; return "BotKleinanzeigenBot";
@ -72,128 +34,7 @@ public class TelegramBot extends TelegramLongPollingBot {
@Override @Override
public void onUpdateReceived(final Update update) { public void onUpdateReceived(final Update update) {
if (update.hasMessage() && update.getMessage().hasText()) { onUpdate.accept(update);
log.info("#{} \"{}\": {}", update.getMessage().getChat().getId(), update.getMessage().getChat().getUserName(), update.getMessage().getText());
} else if (update.hasCallbackQuery()) {
handleCallbackQuery(update.getCallbackQuery());
}
}
private void handleCallbackQuery(final CallbackQuery query) {
final MaybeInaccessibleMessage message = query.getMessage();
if (message.getChatId() != CHAT_ID) {
return;
}
try {
final InlineDto dto = objectMapper.readValue(query.getData(), InlineDto.class);
switch (dto.getCommand()) {
case IGNORE -> ignore(message);
case REMEMBER -> remember(message);
case UNREMEMBER -> unremember(message);
default -> updateMessage(message);
}
} catch (JsonProcessingException e) {
log.error("Failed to read InlineDto.", e);
}
}
private void ignore(final MaybeInaccessibleMessage message) {
offerRepository.ignore(message);
remove(message);
}
private void remember(final MaybeInaccessibleMessage message) {
offerRepository.remember(message, true).ifPresentOrElse(
offer -> updateMessage(message, offer),
() -> remove(message)
);
}
private void unremember(final MaybeInaccessibleMessage message) {
offerRepository.remember(message, false).ifPresentOrElse(
offer -> updateMessage(message, offer),
() -> remove(message)
);
}
private void updateMessage(final MaybeInaccessibleMessage message) {
offerRepository.findByTelegramMessageId(message).ifPresentOrElse(
offer -> updateMessage(message, offer),
() -> remove(message)
);
}
private void updateMessage(final MaybeInaccessibleMessage message, final Offer offer) {
try {
final EditMessageCaption edit = new EditMessageCaption(message.getChatId() + "", message.getMessageId(), null, createText(offer), createKeyboard(offer), null, null);
edit.setParseMode("Markdown");
execute(edit);
} catch (TelegramApiException | JsonProcessingException e) {
log.error("Failed to edit Message to #{}.", message.getChatId(), e);
}
}
public void send(final Offer offer) {
try {
final InputFile inputFile = offer.getImage().isEmpty() ? new InputFile(new ByteArrayInputStream(NO_IMAGE), "[Kein Bild]") : new InputFile(offer.getImage());
final SendPhoto send = new SendPhoto(CHAT_ID + "", inputFile);
send.setCaption(createText(offer));
send.setParseMode("Markdown");
send.setReplyMarkup(createKeyboard(offer));
final Message message = execute(send);
offer.setTelegramMessageId(message.getMessageId());
} catch (TelegramApiException | JsonProcessingException e) {
log.error("Failed to send Message to #{}.", CHAT_ID, e);
}
}
private String createText(final Offer offer) {
return "[%s](%s)\n%s\n%s".formatted(
offer.getTitle().replaceAll("\\[", "(").replaceAll("]", ")"),
offer.getHref(),
offer.calculateLocationString(),
offer.getDescription()
);
}
private InlineKeyboardMarkup createKeyboard(final Offer offer) throws JsonProcessingException {
final InlineKeyboardMarkup markup = new InlineKeyboardMarkup();
final ArrayList<List<InlineKeyboardButton>> keyboard = new ArrayList<>();
final ArrayList<InlineKeyboardButton> row = new ArrayList<>();
if (offer.isRemember()) {
addButton(row, ICON_CHECK + ICON_CHECK + ICON_CHECK + " Gemerkt " + ICON_CHECK + ICON_CHECK + ICON_CHECK, InlineCommand.UNREMEMBER);
} else {
addButton(row, ICON_REMOVE + " Entfernen", InlineCommand.IGNORE);
addButton(row, ICON_CHECK + " Merken", InlineCommand.REMEMBER);
}
keyboard.add(row);
markup.setKeyboard(keyboard);
return markup;
}
private void addButton(final ArrayList<InlineKeyboardButton> row, final String caption, final InlineCommand command) throws JsonProcessingException {
final String data = objectMapper.writeValueAsString(new InlineDto(command));
row.add(new InlineKeyboardButton(caption, null, data, null, null, null, null, null, null));
}
public void remove(final List<Offer> offers) {
_remove(offers.stream().map(Offer::getTelegramMessageId).filter(Objects::nonNull).toList());
}
private void remove(final MaybeInaccessibleMessage... messages) {
_remove(Arrays.stream(messages).map(MaybeInaccessibleMessage::getMessageId).toList());
}
private void _remove(final List<Integer> messageIds) {
if (messageIds.isEmpty()) {
return;
}
try {
execute(new DeleteMessages(CHAT_ID + "", messageIds));
} catch (TelegramApiException e) {
log.error("Failed to remove Message to #{}.", CHAT_ID, e);
}
} }
public void stop() { public void stop() {

View File

@ -0,0 +1,211 @@
package de.ph87.kleinanzeigen.telegram;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferDto;
import de.ph87.kleinanzeigen.kleinanzeigen.offer.OfferService;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Service;
import org.telegram.telegrambots.meta.api.methods.send.SendPhoto;
import org.telegram.telegrambots.meta.api.methods.updatingmessages.DeleteMessages;
import org.telegram.telegrambots.meta.api.methods.updatingmessages.EditMessageCaption;
import org.telegram.telegrambots.meta.api.objects.*;
import org.telegram.telegrambots.meta.api.objects.replykeyboard.InlineKeyboardMarkup;
import org.telegram.telegrambots.meta.api.objects.replykeyboard.buttons.InlineKeyboardButton;
import org.telegram.telegrambots.meta.exceptions.TelegramApiException;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
@Slf4j
@Service
@RequiredArgsConstructor
public class TelegramService {
private static final long CHAT_ID = 101138682L;
private static final String ICON_CHECK = "";
private static final String ICON_REMOVE = "";
private byte[] NO_IMAGE = null;
private final ObjectMapper objectMapper;
private final OfferService offerService;
private TelegramBot bot = null;
@PostConstruct
public void postConstruct() throws IOException {
final BufferedImage img = new BufferedImage(1, 1, BufferedImage.TYPE_INT_ARGB);
final ByteArrayOutputStream stream = new ByteArrayOutputStream();
ImageIO.write(img, "PNG", stream);
NO_IMAGE = stream.toByteArray();
try {
bot = new TelegramBot(readToken(), this::onUpdateReceived);
} catch (TelegramApiException | IOException e) {
log.error("Failed to start TelegramBot: {}", e.toString());
}
}
@EventListener(OfferDto.class)
public void onOfferChanged(final OfferDto offer) {
if (offer.is_existing_()) {
if (offer.getTelegramMessageId() == null) {
send(offer);
}else{
updateMessage(offer.getTelegramMessageId(), offer.getTelegramMessageId().getChatId());
}
} else {
remove(List.of(offer));
}
}
@PreDestroy
public void stop() {
log.info("Stopping Telegram bot...");
bot.stop();
log.info("Telegram bot stopped");
}
private static String readToken() throws IOException {
try (final FileInputStream stream = new FileInputStream("./telegram.token")) {
return new String(stream.readAllBytes(), StandardCharsets.UTF_8).trim();
}
}
private void onUpdateReceived(final Update update) {
if (update.hasMessage() && update.getMessage().hasText()) {
log.info("#{} \"{}\": {}", update.getMessage().getChat().getId(), update.getMessage().getChat().getUserName(), update.getMessage().getText());
} else if (update.hasCallbackQuery()) {
handleCallbackQuery(update.getCallbackQuery());
}
}
private void handleCallbackQuery(final CallbackQuery query) {
final MaybeInaccessibleMessage message = query.getMessage();
if (message.getChatId() != CHAT_ID) {
return;
}
try {
final InlineDto dto = objectMapper.readValue(query.getData(), InlineDto.class);
switch (dto.getCommand()) {
case HIDE -> hide(message);
case REMEMBER -> remember(message, true);
case UNREMEMBER -> remember(message, false);
default -> updateMessage(message, message.getChatId());
}
} catch (JsonProcessingException e) {
log.error("Failed to read InlineDto.", e);
}
}
private void hide(final MaybeInaccessibleMessage message) {
offerService.hideByTelegramMessageId(message.getMessageId(), true);
remove(message);
}
private void remember(final MaybeInaccessibleMessage message, final boolean remember) {
offerService.rememberByTelegramMessageId(message.getMessageId(), remember).ifPresentOrElse(
offer -> updateMessage(offer, message.getChatId(), message.getMessageId()),
() -> remove(message)
);
}
private void updateMessage(final MaybeInaccessibleMessage message, final Long chatId) {
offerService.findByTelegramMessageId(message.getMessageId()).ifPresentOrElse(
offer -> updateMessage(offer, chatId, message.getMessageId()),
() -> remove(message)
);
}
private void updateMessage(final OfferDto offer, final long chatId, final int messageId) {
try {
final EditMessageCaption edit = new EditMessageCaption(chatId + "", messageId, null, createText(offer), createKeyboard(offer), null, null);
edit.setParseMode("Markdown");
bot.execute(edit);
} catch (TelegramApiException | JsonProcessingException e) {
log.error("Failed to edit Message to #{}.", chatId, e);
}
}
private void send(final OfferDto offer) {
try {
final InputFile inputFile = offer.getImageURL() == null ? new InputFile(new ByteArrayInputStream(NO_IMAGE), "[Kein Bild]") : new InputFile(offer.getImageURL());
final SendPhoto send = new SendPhoto(CHAT_ID + "", inputFile);
log.debug("{}", send);
send.setCaption(createText(offer));
send.setParseMode("Markdown");
send.setReplyMarkup(createKeyboard(offer));
final Message message = bot.execute(send);
offerService.setTelegramMessageId(offer, message.getMessageId());
} catch (TelegramApiException | JsonProcessingException e) {
log.error("Failed to send Message to #{}.", CHAT_ID, e);
}
}
private String createText(final OfferDto offer) {
return "[%s](%s)\n%s\n%s".formatted(
offer.getTitle().replaceAll("\\[", "(").replaceAll("]", ")"),
offer.getArticleURL(),
offer.combineLocation(),
offer.getDescription()
);
}
private InlineKeyboardMarkup createKeyboard(final OfferDto offer) throws JsonProcessingException {
final InlineKeyboardMarkup markup = new InlineKeyboardMarkup();
final ArrayList<List<InlineKeyboardButton>> keyboard = new ArrayList<>();
final ArrayList<InlineKeyboardButton> row = new ArrayList<>();
if (offer.isRemember()) {
addButton(row, ICON_CHECK + ICON_CHECK + ICON_CHECK + " Gemerkt " + ICON_CHECK + ICON_CHECK + ICON_CHECK, InlineCommand.UNREMEMBER);
} else {
addButton(row, ICON_REMOVE + " Entfernen", InlineCommand.HIDE);
addButton(row, ICON_CHECK + " Merken", InlineCommand.REMEMBER);
}
keyboard.add(row);
markup.setKeyboard(keyboard);
return markup;
}
private void addButton(final ArrayList<InlineKeyboardButton> row, final String caption, final InlineCommand command) throws JsonProcessingException {
final String data = objectMapper.writeValueAsString(new InlineDto(command));
row.add(new InlineKeyboardButton(caption, null, data, null, null, null, null, null, null));
}
private void remove(final List<OfferDto> offers) {
_remove(offers.stream().map(OfferDto::getTelegramMessageId).filter(Objects::nonNull).toList());
}
private void remove(final MaybeInaccessibleMessage... messages) {
_remove(Arrays.stream(messages).map(MaybeInaccessibleMessage::getMessageId).toList());
}
private void _remove(final List<Integer> messageIds) {
if (messageIds.isEmpty()) {
return;
}
try {
bot.execute(new DeleteMessages(CHAT_ID + "", messageIds));
} catch (TelegramApiException e) {
log.error("Failed to remove Message to #{}.", CHAT_ID, e);
}
}
}

View File

@ -0,0 +1,10 @@
logging.level.root=WARN
logging.level.de.ph87=INFO
#-
spring.jpa.hibernate.naming.implicit-strategy=org.hibernate.boot.model.naming.ImplicitNamingStrategyComponentPathImpl
spring.jpa.hibernate.ddl-auto=update
spring.jpa.open-in-view=false
#-
spring.jackson.serialization.indent_output=true
#-
spring.main.banner-mode=off