71 lines
2.3 KiB
C#
71 lines
2.3 KiB
C#
using System.Diagnostics;
|
|
using Minio;
|
|
using Minio.DataModel.Args;
|
|
using Npgsql;
|
|
using skyscraper5.Data;
|
|
using skyscraper5.Data.PostgreSql;
|
|
using skyscraper5.Skyscraper.Plugins;
|
|
using skyscraper5.Skyscraper.Scraper.Storage;
|
|
using skyscraper5.Skyscraper.Scraper.Storage.Split;
|
|
|
|
namespace skyscraper5.Storage.PostgresqlMinio
|
|
{
|
|
[SkyscraperPlugin]
|
|
[ScrapeStorageFactoryId(3,"PostgreSQL & MinIO",false)]
|
|
public class PostgresqlMinioStorageFactory : IScraperStorageFactory
|
|
{
|
|
public IScraperStroage CreateScraperStroage()
|
|
{
|
|
NpgsqlConnectionStringBuilder ncsb = new NpgsqlConnectionStringBuilder();
|
|
ncsb.Database = PostgreSqlDatabase;
|
|
ncsb.ApplicationName = "skyscraper5";
|
|
ncsb.Host = PostgreSqlHost;
|
|
ncsb.Password = PostgreSqlPassword;
|
|
ncsb.Pooling = true;
|
|
ncsb.Port = PostgreSqlPort;
|
|
ncsb.Username = PostgreSqlUsername;
|
|
|
|
if (Debugger.IsAttached)
|
|
ncsb.Timeout = 1024;
|
|
|
|
IMinioClient mc = new MinioClient()
|
|
.WithEndpoint(MinioEndpoint)
|
|
.WithCredentials(MinioAccessKey, MinioSecretKey)
|
|
.WithSSL(MinioSecure).Build();
|
|
BucketExistsArgs bucketExistsArgs = new BucketExistsArgs().WithBucket(MinioBucket);
|
|
bool bucketExists = mc.BucketExistsAsync(bucketExistsArgs).Result;
|
|
if (!bucketExists)
|
|
{
|
|
MakeBucketArgs makeBucketArgs = new MakeBucketArgs().WithBucket(MinioBucket);
|
|
mc.MakeBucketAsync(makeBucketArgs).Wait();
|
|
}
|
|
|
|
PostgresqlDataStore postgresDs = new PostgresqlDataStore(ncsb);
|
|
MinioObjectStorage minioOs = new MinioObjectStorage(mc, MinioBucket);
|
|
|
|
|
|
return new SplitScraperStorage(postgresDs, minioOs);
|
|
}
|
|
|
|
public string MinioBucket { get; set; }
|
|
|
|
public bool MinioSecure { get; set; }
|
|
|
|
public string MinioSecretKey { get; set; }
|
|
|
|
public string MinioAccessKey { get; set; }
|
|
|
|
public string MinioEndpoint { get; set; }
|
|
|
|
public string PostgreSqlUsername { get; set; }
|
|
|
|
public ushort PostgreSqlPort { get; set; }
|
|
|
|
public string PostgreSqlPassword { get; set; }
|
|
|
|
public string PostgreSqlHost { get; set; }
|
|
|
|
public string PostgreSqlDatabase { get; set; }
|
|
}
|
|
}
|